예제 #1
0
def set_environment(app, elem, tool_shed_repository):
    """
    Create a ToolDependency to set an environment variable.  This is different from the process used to set an environment variable that is associated
    with a package.  An example entry in a tool_dependencies.xml file is::

        <set_environment version="1.0">
            <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
        </set_environment>
    """
    # TODO: Add support for a repository dependency definition within this tool dependency type's tag set.  This should look something like
    # the following.  See the implementation of support for this in the tool dependency package type's method above.
    # <set_environment version="1.0">
    #    <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
    # </set_environment>
    sa_session = app.model.context.current
    tool_dependency = None
    env_var_version = elem.get("version", "1.0")
    for env_var_elem in elem:
        # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
        # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
        env_var_name = env_var_elem.get("name", None)
        env_var_action = env_var_elem.get("action", None)
        if env_var_name and env_var_action:
            install_dir = get_tool_dependency_install_dir(
                app=app,
                repository_name=tool_shed_repository.name,
                repository_owner=tool_shed_repository.owner,
                repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
                tool_dependency_type="set_environment",
                tool_dependency_name=env_var_name,
                tool_dependency_version=None,
            )
            tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(app, tool_shed_repository)
            env_var_dict = common_util.create_env_var_dict(
                env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir
            )
            if env_var_dict:
                if not os.path.exists(install_dir):
                    os.makedirs(install_dir)
                tool_dependency = create_or_update_tool_dependency(
                    app=app,
                    tool_shed_repository=tool_shed_repository,
                    name=env_var_name,
                    version=None,
                    type="set_environment",
                    status=app.model.ToolDependency.installation_status.INSTALLING,
                    set_status=True,
                )
                cmd = common_util.create_or_update_env_shell_file(install_dir, env_var_dict)
                if env_var_version == "1.0":
                    # Handle setting environment variables using a fabric method.
                    fabric_util.handle_command(app, tool_dependency, install_dir, cmd)
                    sa_session.refresh(tool_dependency)
                    if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
                        tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
                        sa_session.add(tool_dependency)
                        sa_session.flush()
                        print "Environment variable ", env_var_name, "set in", install_dir
예제 #2
0
def handle_set_environment_entry_for_package(app, install_dir,
                                             tool_shed_repository,
                                             package_name, package_version,
                                             elem):
    action_dict = {}
    actions = []
    for package_elem in elem:
        if package_elem.tag == 'install':
            # Create the tool_dependency record in the database.
            tool_dependency = create_or_update_tool_dependency(
                app=app,
                tool_shed_repository=tool_shed_repository,
                name=package_name,
                version=package_version,
                type='package',
                status=app.model.ToolDependency.installation_status.INSTALLING,
                set_status=True)
            # Get the installation method version from a tag like: <install version="1.0">
            package_install_version = package_elem.get('version', '1.0')
            if package_install_version == '1.0':
                # Since the required tool dependency is installed for a repository dependency, all we need to do
                # is inspect the <actions> tag set to find the <action type="set_environment"> tag.
                for actions_elem in package_elem:
                    for action_elem in actions_elem:
                        action_type = action_elem.get('type', 'shell_command')
                        if action_type == 'set_environment':
                            # <action type="set_environment">
                            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
                            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                            # </action>
                            env_var_dicts = []
                            for env_elem in action_elem:
                                if env_elem.tag == 'environment_variable':
                                    env_var_dict = common_util.create_env_var_dict(
                                        env_elem,
                                        tool_dependency_install_dir=install_dir
                                    )
                                    if env_var_dict:
                                        env_var_dicts.append(env_var_dict)
                            if env_var_dicts:
                                action_dict[env_elem.tag] = env_var_dicts
                                actions.append((action_type, action_dict))
                            return tool_dependency, actions
    return None, actions
예제 #3
0
def handle_set_environment_entry_for_package(
    app, install_dir, tool_shed_repository, package_name, package_version, elem
):
    action_dict = {}
    actions = []
    for package_elem in elem:
        if package_elem.tag == "install":
            # Create the tool_dependency record in the database.
            tool_dependency = create_or_update_tool_dependency(
                app=app,
                tool_shed_repository=tool_shed_repository,
                name=package_name,
                version=package_version,
                type="package",
                status=app.model.ToolDependency.installation_status.INSTALLING,
                set_status=True,
            )
            # Get the installation method version from a tag like: <install version="1.0">
            package_install_version = package_elem.get("version", "1.0")
            if package_install_version == "1.0":
                # Since the required tool dependency is installed for a repository dependency, all we need to do
                # is inspect the <actions> tag set to find the <action type="set_environment"> tag.
                for actions_elem in package_elem:
                    for action_elem in actions_elem:
                        action_type = action_elem.get("type", "shell_command")
                        if action_type == "set_environment":
                            # <action type="set_environment">
                            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
                            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                            # </action>
                            env_var_dicts = []
                            for env_elem in action_elem:
                                if env_elem.tag == "environment_variable":
                                    env_var_dict = common_util.create_env_var_dict(
                                        env_elem, tool_dependency_install_dir=install_dir
                                    )
                                    if env_var_dict:
                                        env_var_dicts.append(env_var_dict)
                            if env_var_dicts:
                                action_dict[env_elem.tag] = env_var_dicts
                                actions.append((action_type, action_dict))
                            return tool_dependency, actions
    return None, actions
예제 #4
0
def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd ):
    """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
    sa_session = app.model.context.current

    def evaluate_template( text ):
        """ Substitute variables defined in XML blocks from dependencies file."""
        return Template( text ).safe_substitute( common_util.get_env_var_values( install_dir ) )

    if not os.path.exists( install_dir ):
        os.makedirs( install_dir )
    actions_dict = dict( install_dir=install_dir )
    if package_name:
        actions_dict[ 'package_name' ] = package_name
    actions = []
    all_env_shell_file_paths = []
    env_var_dicts = []
    # Make sure to skip all comments, since they are now included in the XML tree.
    for action_elem in actions_elem.findall( 'action' ):
        action_dict = {}
        action_type = action_elem.get( 'type', 'shell_command' )
        if action_type == 'shell_command':
            # <action type="shell_command">make</action>
            action_elem_text = evaluate_template( action_elem.text )
            if action_elem_text:
                action_dict[ 'command' ] = action_elem_text
            else:
                continue
        elif action_type == 'template_command':
            # Default to Cheetah as it's the first template language supported.
            language = action_elem.get( 'language', 'cheetah' ).lower()
            if language == 'cheetah':
                # Cheetah template syntax.
                # <action type="template_command" language="cheetah">
                #     #if env.PATH:
                #         make
                #     #end if
                # </action>
                action_elem_text = action_elem.text.strip()
                if action_elem_text:
                    action_dict[ 'language' ] = language
                    action_dict[ 'command' ] = action_elem_text
                else:
                    continue
            else:
                log.debug( "Unsupported template language '%s'. Not proceeding." % str( language ) )
                raise Exception( "Unsupported template language '%s' in tool dependency definition." % str( language ) )
        elif action_type == 'download_by_url':
            # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
            if action_elem.text:
                action_dict[ 'url' ] = action_elem.text
                target_filename = action_elem.get( 'target_filename', None )
                if target_filename:
                    action_dict[ 'target_filename' ] = target_filename
            else:
                continue
        elif action_type == 'download_file':
            # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
            if action_elem.text:
                action_dict[ 'url' ] = action_elem.text
                target_filename = action_elem.get( 'target_filename', None )
                if target_filename:
                    action_dict[ 'target_filename' ] = target_filename
                action_dict[ 'extract' ] = asbool( action_elem.get( 'extract', False ) )
            else:
                continue
        elif action_type == 'make_directory':
            # <action type="make_directory">$INSTALL_DIR/lib/python</action>
            if action_elem.text:
                action_dict[ 'full_path' ] = evaluate_template( action_elem.text )
            else:
                continue
        elif action_type == 'change_directory':
            # <action type="change_directory">PHYLIP-3.6b</action>
            if action_elem.text:
                action_dict[ 'directory' ] = action_elem.text
            else:
                continue
        elif action_type in [ 'move_directory_files', 'move_file' ]:
            # <action type="move_file">
            #     <source>misc/some_file</source>
            #     <destination>$INSTALL_DIR/bin</destination>
            # </action>
            # <action type="move_directory_files">
            #     <source_directory>bin</source_directory>
            #     <destination_directory>$INSTALL_DIR/bin</destination_directory>
            # </action>
            for move_elem in action_elem:
                move_elem_text = evaluate_template( move_elem.text )
                if move_elem_text:
                    action_dict[ move_elem.tag ] = move_elem_text
        elif action_type == 'set_environment':
            # <action type="set_environment">
            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
            # </action>
            for env_elem in action_elem:
                if env_elem.tag == 'environment_variable':
                    env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir )
                    if env_var_dict:
                        env_var_dicts.append( env_var_dict )
            if env_var_dicts:
                # The last child of an <action type="set_environment"> might be a comment, so manually set it to be 'environment_variable'.
                action_dict[ 'environment_variable' ] = env_var_dicts
            else:
                continue
        elif action_type == 'set_environment_for_install':
            # <action type="set_environment_for_install">
            #    <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
            #        <package name="numpy" version="1.7.1" />
            #    </repository>
            # </action>
            # This action type allows for defining an environment that will properly compile a tool dependency.  Currently, tag set definitions like
            # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be 
            # supported.  The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current
            # tool dependency package.  See the package_matplotlib_1_2 repository in the test tool shed for a real-world example.
            for env_elem in action_elem:
                if env_elem.tag == 'repository':
                    env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem )
                    if env_shell_file_paths:
                        all_env_shell_file_paths.extend( env_shell_file_paths )
            if all_env_shell_file_paths:
                action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
            else:
                continue
        elif action_type == 'setup_virtualenv':
            # <action type="setup_virtualenv" />
            ## Install requirements from file requirements.txt of downloaded bundle - or -
            # <action type="setup_virtualenv">tools/requirements.txt</action>
            ## Install requirements from specified file from downloaded bundle -or -
            # <action type="setup_virtualenv">pyyaml==3.2.0
            # lxml==2.3.0</action>
            ## Manually specify contents of requirements.txt file to create dynamically.
            action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' )
        else:
            log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) )
            raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) )
        action_tuple = ( action_type, action_dict )
        if action_type == 'set_environment':
            if action_tuple not in actions:
                actions.append( action_tuple )
        else:
            actions.append( action_tuple )
    if actions:
        actions_dict[ 'actions' ] = actions
    if proprietary_fabfile_path:
        # TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
        # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
        raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' )
    else:
        install_and_build_package_via_fabric( app, tool_dependency, actions_dict )
예제 #5
0
def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ):
    """
    Populate a list of actions for creating an env.sh file for a dependent repository.  The received elem is the <package> tag set associated
    with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
    """
    action_dict = {}
    actions = []
    for package_elem in elem:
        if package_elem.tag == 'install':
            # Create the tool_dependency record in the database.
            tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app,
                                                                                     tool_shed_repository=tool_shed_repository,
                                                                                     name=package_name,
                                                                                     version=package_version,
                                                                                     type='package',
                                                                                     status=app.model.ToolDependency.installation_status.INSTALLING,
                                                                                     set_status=True )
            # Get the installation method version from a tag like: <install version="1.0">
            package_install_version = package_elem.get( 'version', '1.0' )
            if package_install_version == '1.0':
                # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
                # the <action type="set_environment"> tag.
                env_var_dicts = []
                for actions_elem in package_elem:
                    for action_elem in actions_elem:
                        action_type = action_elem.get( 'type', 'shell_command' )
                        if action_type == 'set_environment':
                            # <action type="set_environment">
                            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
                            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                            # </action>
                            for env_elem in action_elem:
                                if env_elem.tag == 'environment_variable':
                                    env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir )
                                    if env_var_dict:
                                        if env_var_dict not in env_var_dicts:
                                            env_var_dicts.append( env_var_dict )
                        elif action_type == 'setup_virtualenv':
                            # Add the virtualenv's site-packages to PYTHONPATH and bin to PATH.  This is a bit hackish.
                            site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" )
                            output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
                            if output.return_code:
                                log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr )
                            elif not os.path.exists( output.stdout ):
                                log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
                            else:
                                env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
                                env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) )
                if env_var_dicts:
                    if required_repository.status in [ app.model.ToolShedRepository.installation_status.INSTALLED,
                                                       app.model.ToolShedRepository.installation_status.DEACTIVATED ]:
                        # Handle the case where we have an installed required repository due to the prior_installation_required = True
                        # setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
                        # tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature.  To handle this,
                        # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
                        # file generated for the installed required repository.  Each env_var_dict currently looks something like this: 
                        # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
                        # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
                        # with the associated value in the env.sh file.
                        new_env_var_dicts = []
                        env_sh_file_dir = get_tool_dependency_install_dir( app=app,
                                                                           repository_name=required_repository.name,
                                                                           repository_owner=required_repository.owner,
                                                                           repository_changeset_revision=required_repository.installed_changeset_revision,
                                                                           tool_dependency_type='package',
                                                                           tool_dependency_name=package_name,
                                                                           tool_dependency_version=package_version )
                        env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
                        if os.path.exists( env_sh_file_path ):
                            for i, line in enumerate( open( env_sh_file_path, 'r' ) ):
                                env_var_dict = env_var_dicts[ i ]
                                action = env_var_dict.get( 'action', None )
                                name = env_var_dict.get( 'name', None )
                                value = env_var_dict.get( 'value', None )
                                if action and name and value:
                                    new_value = parse_env_shell_entry( action, name, value, line )
                                    env_var_dict[ 'value' ] = new_value
                                new_env_var_dicts.append( env_var_dict )
                        else:
                            log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path )
                        action_dict[ 'environment_variable' ] = new_env_var_dicts
                    else:
                        action_dict[ 'environment_variable' ] = env_var_dicts
                    actions.append( ( 'set_environment', action_dict ) )
            else:
                raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
            return tool_dependency, actions
    return None, actions
예제 #6
0
def set_environment(app, elem, tool_shed_repository):
    """
    Create a ToolDependency to set an environment variable.  This is different from the process used to set an environment variable that is associated
    with a package.  An example entry in a tool_dependencies.xml file is::

        <set_environment version="1.0">
            <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
        </set_environment>
    """
    # TODO: Add support for a repository dependency definition within this tool dependency type's tag set.  This should look something like
    # the following.  See the implementation of support for this in the tool dependency package type's method above.
    # <set_environment version="1.0">
    #    <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
    # </set_environment>
    sa_session = app.model.context.current
    tool_dependency = None
    env_var_version = elem.get('version', '1.0')
    for env_var_elem in elem:
        # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
        # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
        env_var_name = env_var_elem.get('name', None)
        env_var_action = env_var_elem.get('action', None)
        if env_var_name and env_var_action:
            install_dir = get_tool_dependency_install_dir(
                app=app,
                repository_name=tool_shed_repository.name,
                repository_owner=tool_shed_repository.owner,
                repository_changeset_revision=tool_shed_repository.
                installed_changeset_revision,
                tool_dependency_type='set_environment',
                tool_dependency_name=env_var_name,
                tool_dependency_version=None)
            tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(
                app, tool_shed_repository)
            env_var_dict = common_util.create_env_var_dict(
                env_var_elem,
                tool_shed_repository_install_dir=
                tool_shed_repository_install_dir)
            if env_var_dict:
                if not os.path.exists(install_dir):
                    os.makedirs(install_dir)
                tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
                    app=app,
                    tool_shed_repository=tool_shed_repository,
                    name=env_var_name,
                    version=None,
                    type='set_environment',
                    status=app.model.ToolDependency.installation_status.
                    INSTALLING,
                    set_status=True)
                cmd = common_util.create_or_update_env_shell_file(
                    install_dir, env_var_dict)
                if env_var_version == '1.0':
                    # Handle setting environment variables using a fabric method.
                    fabric_util.handle_command(app, tool_dependency,
                                               install_dir, cmd)
                    sa_session.refresh(tool_dependency)
                    if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
                        tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
                        sa_session.add(tool_dependency)
                        sa_session.flush()
                        print 'Environment variable ', env_var_name, 'set in', install_dir
                else:
                    raise NotImplementedError(
                        'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
                    )
예제 #7
0
def install_via_fabric(app,
                       tool_dependency,
                       actions_elem,
                       install_dir,
                       package_name=None,
                       proprietary_fabfile_path=None,
                       **kwd):
    """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
    sa_session = app.model.context.current

    def evaluate_template(text):
        """ Substitute variables defined in XML blocks from dependencies file."""
        return Template(text).safe_substitute(
            common_util.get_env_var_values(install_dir))

    if not os.path.exists(install_dir):
        os.makedirs(install_dir)
    actions_dict = dict(install_dir=install_dir)
    if package_name:
        actions_dict['package_name'] = package_name
    actions = []
    all_env_shell_file_paths = []
    env_var_dicts = []
    # Make sure to skip all comments, since they are now included in the XML tree.
    for action_elem in actions_elem.findall('action'):
        action_dict = {}
        action_type = action_elem.get('type', 'shell_command')
        if action_type == 'shell_command':
            # <action type="shell_command">make</action>
            action_elem_text = evaluate_template(action_elem.text)
            if action_elem_text:
                action_dict['command'] = action_elem_text
            else:
                continue
        elif action_type == 'template_command':
            # Default to Cheetah as it's the first template language supported.
            language = action_elem.get('language', 'cheetah').lower()
            if language == 'cheetah':
                # Cheetah template syntax.
                # <action type="template_command" language="cheetah">
                #     #if env.PATH:
                #         make
                #     #end if
                # </action>
                action_elem_text = action_elem.text.strip()
                if action_elem_text:
                    action_dict['language'] = language
                    action_dict['command'] = action_elem_text
                else:
                    continue
            else:
                log.debug(
                    "Unsupported template language '%s'. Not proceeding." %
                    str(language))
                raise Exception(
                    "Unsupported template language '%s' in tool dependency definition."
                    % str(language))
        elif action_type == 'download_by_url':
            # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
            if action_elem.text:
                action_dict['url'] = action_elem.text
                target_filename = action_elem.get('target_filename', None)
                if target_filename:
                    action_dict['target_filename'] = target_filename
            else:
                continue
        elif action_type == 'download_file':
            # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
            if action_elem.text:
                action_dict['url'] = action_elem.text
                target_filename = action_elem.get('target_filename', None)
                if target_filename:
                    action_dict['target_filename'] = target_filename
                action_dict['extract'] = asbool(
                    action_elem.get('extract', False))
            else:
                continue
        elif action_type == 'make_directory':
            # <action type="make_directory">$INSTALL_DIR/lib/python</action>
            if action_elem.text:
                action_dict['full_path'] = evaluate_template(action_elem.text)
            else:
                continue
        elif action_type == 'change_directory':
            # <action type="change_directory">PHYLIP-3.6b</action>
            if action_elem.text:
                action_dict['directory'] = action_elem.text
            else:
                continue
        elif action_type in ['move_directory_files', 'move_file']:
            # <action type="move_file">
            #     <source>misc/some_file</source>
            #     <destination>$INSTALL_DIR/bin</destination>
            # </action>
            # <action type="move_directory_files">
            #     <source_directory>bin</source_directory>
            #     <destination_directory>$INSTALL_DIR/bin</destination_directory>
            # </action>
            for move_elem in action_elem:
                move_elem_text = evaluate_template(move_elem.text)
                if move_elem_text:
                    action_dict[move_elem.tag] = move_elem_text
        elif action_type == 'set_environment':
            # <action type="set_environment">
            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
            # </action>
            for env_elem in action_elem:
                if env_elem.tag == 'environment_variable':
                    env_var_dict = common_util.create_env_var_dict(
                        env_elem, tool_dependency_install_dir=install_dir)
                    if env_var_dict:
                        env_var_dicts.append(env_var_dict)
            if env_var_dicts:
                # The last child of an <action type="set_environment"> might be a comment, so manually set it to be 'environment_variable'.
                action_dict['environment_variable'] = env_var_dicts
            else:
                continue
        elif action_type == 'set_environment_for_install':
            # <action type="set_environment_for_install">
            #    <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
            #        <package name="numpy" version="1.7.1" />
            #    </repository>
            # </action>
            # This action type allows for defining an environment that will properly compile a tool dependency.  Currently, tag set definitions like
            # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be
            # supported.  The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current
            # tool dependency package.  See the package_matplotlib_1_2 repository in the test tool shed for a real-world example.
            for env_elem in action_elem:
                if env_elem.tag == 'repository':
                    env_shell_file_paths = common_util.get_env_shell_file_paths(
                        app, env_elem)
                    if env_shell_file_paths:
                        all_env_shell_file_paths.extend(env_shell_file_paths)
            if all_env_shell_file_paths:
                action_dict['env_shell_file_paths'] = all_env_shell_file_paths
            else:
                continue
        elif action_type == 'setup_virtualenv':
            # <action type="setup_virtualenv" />
            ## Install requirements from file requirements.txt of downloaded bundle - or -
            # <action type="setup_virtualenv">tools/requirements.txt</action>
            ## Install requirements from specified file from downloaded bundle -or -
            # <action type="setup_virtualenv">pyyaml==3.2.0
            # lxml==2.3.0</action>
            ## Manually specify contents of requirements.txt file to create dynamically.
            action_dict['requirements'] = evaluate_template(
                action_elem.text or 'requirements.txt')
        else:
            log.debug("Unsupported action type '%s'. Not proceeding." %
                      str(action_type))
            raise Exception(
                "Unsupported action type '%s' in tool dependency definition." %
                str(action_type))
        action_tuple = (action_type, action_dict)
        if action_type == 'set_environment':
            if action_tuple not in actions:
                actions.append(action_tuple)
        else:
            actions.append(action_tuple)
    if actions:
        actions_dict['actions'] = actions
    if proprietary_fabfile_path:
        # TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
        # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
        raise Exception(
            'Tool dependency installation using proprietary fabric scripts is not yet supported.'
        )
    else:
        install_and_build_package_via_fabric(app, tool_dependency,
                                             actions_dict)
예제 #8
0
def handle_set_environment_entry_for_package(app, install_dir,
                                             tool_shed_repository,
                                             package_name, package_version,
                                             elem, required_repository):
    """
    Populate a list of actions for creating an env.sh file for a dependent repository.  The received elem is the <package> tag set associated
    with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
    """
    action_dict = {}
    actions = []
    for package_elem in elem:
        if package_elem.tag == 'install':
            # Create the tool_dependency record in the database.
            tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
                app=app,
                tool_shed_repository=tool_shed_repository,
                name=package_name,
                version=package_version,
                type='package',
                status=app.model.ToolDependency.installation_status.INSTALLING,
                set_status=True)
            # Get the installation method version from a tag like: <install version="1.0">
            package_install_version = package_elem.get('version', '1.0')
            if package_install_version == '1.0':
                # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
                # the <action type="set_environment"> tag.
                env_var_dicts = []
                for actions_elem in package_elem:
                    for action_elem in actions_elem:
                        action_type = action_elem.get('type', 'shell_command')
                        if action_type == 'set_environment':
                            # <action type="set_environment">
                            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
                            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                            # </action>
                            for env_elem in action_elem:
                                if env_elem.tag == 'environment_variable':
                                    env_var_dict = common_util.create_env_var_dict(
                                        env_elem,
                                        tool_dependency_install_dir=install_dir
                                    )
                                    if env_var_dict:
                                        if env_var_dict not in env_var_dicts:
                                            env_var_dicts.append(env_var_dict)
                        elif action_type == 'setup_virtualenv':
                            # Add the virtualenv's site-packages to PYTHONPATH and bin to PATH.  This is a bit hackish.
                            site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join(
                                install_dir, "venv", "bin", "python")
                            output = fabric_util.handle_command(
                                app,
                                tool_dependency,
                                install_dir,
                                site_packages_command,
                                return_output=True)
                            if output.return_code:
                                log.error(
                                    'Dependency includes a setup_virtualenv action but venv python is broken:',
                                    output.stderr)
                            elif not os.path.exists(output.stdout):
                                log.error(
                                    "virtualenv's site-packages directory '%s' does not exist",
                                    output.stdout)
                            else:
                                env_var_dicts.append(
                                    dict(name="PYTHONPATH",
                                         action="prepend_to",
                                         value=output.stdout))
                                env_var_dicts.append(
                                    dict(name="PATH",
                                         action="prepend_to",
                                         value=os.path.join(
                                             install_dir, 'venv', 'bin')))
                if env_var_dicts:
                    if required_repository.status in [
                            app.model.ToolShedRepository.installation_status.
                            INSTALLED, app.model.ToolShedRepository.
                            installation_status.DEACTIVATED
                    ]:
                        # Handle the case where we have an installed required repository due to the prior_installation_required = True
                        # setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
                        # tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature.  To handle this,
                        # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
                        # file generated for the installed required repository.  Each env_var_dict currently looks something like this:
                        # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
                        # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
                        # with the associated value in the env.sh file.
                        new_env_var_dicts = []
                        env_sh_file_dir = get_tool_dependency_install_dir(
                            app=app,
                            repository_name=required_repository.name,
                            repository_owner=required_repository.owner,
                            repository_changeset_revision=required_repository.
                            installed_changeset_revision,
                            tool_dependency_type='package',
                            tool_dependency_name=package_name,
                            tool_dependency_version=package_version)
                        env_sh_file_path = os.path.join(
                            env_sh_file_dir, 'env.sh')
                        if os.path.exists(env_sh_file_path):
                            for i, line in enumerate(
                                    open(env_sh_file_path, 'r')):
                                env_var_dict = env_var_dicts[i]
                                action = env_var_dict.get('action', None)
                                name = env_var_dict.get('name', None)
                                value = env_var_dict.get('value', None)
                                if action and name and value:
                                    new_value = parse_env_shell_entry(
                                        action, name, value, line)
                                    env_var_dict['value'] = new_value
                                new_env_var_dicts.append(env_var_dict)
                        else:
                            log.debug(
                                'Invalid file %s specified, ignoring set_environment_for_install action.',
                                env_sh_file_path)
                        action_dict['environment_variable'] = new_env_var_dicts
                    else:
                        action_dict['environment_variable'] = env_var_dicts
                    actions.append(('set_environment', action_dict))
            else:
                raise NotImplementedError(
                    'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
                )
            return tool_dependency, actions
    return None, actions
예제 #9
0
def install_via_fabric(app,
                       tool_dependency,
                       actions_elem,
                       install_dir,
                       package_name=None,
                       proprietary_fabfile_path=None,
                       **kwd):
    """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
    sa_session = app.model.context.current
    if not os.path.exists(install_dir):
        os.makedirs(install_dir)
    actions_dict = dict(install_dir=install_dir)
    if package_name:
        actions_dict['package_name'] = package_name
    actions = []
    for action_elem in actions_elem:
        action_dict = {}
        action_type = action_elem.get('type', 'shell_command')
        if action_type == 'shell_command':
            # <action type="shell_command">make</action>
            action_elem_text = action_elem.text.replace(
                '$INSTALL_DIR', install_dir)
            if action_elem_text:
                action_dict['command'] = action_elem_text
            else:
                continue
        elif action_type == 'download_by_url':
            # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
            if action_elem.text:
                action_dict['url'] = action_elem.text
                if 'target_filename' in action_elem.attrib:
                    action_dict['target_filename'] = action_elem.attrib[
                        'target_filename']
            else:
                continue
        elif action_type == 'make_directory':
            # <action type="make_directory">$INSTALL_DIR/lib/python</action>
            if action_elem.text:
                action_dict['full_path'] = action_elem.text.replace(
                    '$INSTALL_DIR', install_dir)
            else:
                continue
        elif action_type in ['move_directory_files', 'move_file']:
            # <action type="move_file">
            #     <source>misc/some_file</source>
            #     <destination>$INSTALL_DIR/bin</destination>
            # </action>
            # <action type="move_directory_files">
            #     <source_directory>bin</source_directory>
            #     <destination_directory>$INSTALL_DIR/bin</destination_directory>
            # </action>
            for move_elem in action_elem:
                move_elem_text = move_elem.text.replace(
                    '$INSTALL_DIR', install_dir)
                if move_elem_text:
                    action_dict[move_elem.tag] = move_elem_text
        elif action_type == 'set_environment':
            # <action type="set_environment">
            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
            # </action>
            env_var_dicts = []
            for env_elem in action_elem:
                if env_elem.tag == 'environment_variable':
                    env_var_dict = common_util.create_env_var_dict(
                        env_elem, tool_dependency_install_dir=install_dir)
                    if env_var_dict:
                        env_var_dicts.append(env_var_dict)
            if env_var_dicts:
                action_dict[env_elem.tag] = env_var_dicts
            else:
                continue
        else:
            continue
        actions.append((action_type, action_dict))
    if actions:
        actions_dict['actions'] = actions
    if proprietary_fabfile_path:
        # TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
        # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
        raise Exception(
            'Tool dependency installation using proprietary fabric scripts is not yet supported.'
        )
    else:
        install_and_build_package_via_fabric(app, tool_dependency,
                                             actions_dict)
예제 #10
0
def install_via_fabric(
    app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd
):
    """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
    sa_session = app.model.context.current
    if not os.path.exists(install_dir):
        os.makedirs(install_dir)
    actions_dict = dict(install_dir=install_dir)
    if package_name:
        actions_dict["package_name"] = package_name
    actions = []
    for action_elem in actions_elem:
        action_dict = {}
        action_type = action_elem.get("type", "shell_command")
        if action_type == "shell_command":
            # <action type="shell_command">make</action>
            action_elem_text = action_elem.text.replace("$INSTALL_DIR", install_dir)
            if action_elem_text:
                action_dict["command"] = action_elem_text
            else:
                continue
        elif action_type == "download_by_url":
            # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
            if action_elem.text:
                action_dict["url"] = action_elem.text
                if "target_filename" in action_elem.attrib:
                    action_dict["target_filename"] = action_elem.attrib["target_filename"]
            else:
                continue
        elif action_type == "make_directory":
            # <action type="make_directory">$INSTALL_DIR/lib/python</action>
            if action_elem.text:
                action_dict["full_path"] = action_elem.text.replace("$INSTALL_DIR", install_dir)
            else:
                continue
        elif action_type in ["move_directory_files", "move_file"]:
            # <action type="move_file">
            #     <source>misc/some_file</source>
            #     <destination>$INSTALL_DIR/bin</destination>
            # </action>
            # <action type="move_directory_files">
            #     <source_directory>bin</source_directory>
            #     <destination_directory>$INSTALL_DIR/bin</destination_directory>
            # </action>
            for move_elem in action_elem:
                move_elem_text = move_elem.text.replace("$INSTALL_DIR", install_dir)
                if move_elem_text:
                    action_dict[move_elem.tag] = move_elem_text
        elif action_type == "set_environment":
            # <action type="set_environment">
            #     <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
            #     <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
            # </action>
            env_var_dicts = []
            for env_elem in action_elem:
                if env_elem.tag == "environment_variable":
                    env_var_dict = common_util.create_env_var_dict(env_elem, tool_dependency_install_dir=install_dir)
                    if env_var_dict:
                        env_var_dicts.append(env_var_dict)
            if env_var_dicts:
                action_dict[env_elem.tag] = env_var_dicts
            else:
                continue
        else:
            continue
        actions.append((action_type, action_dict))
    if actions:
        actions_dict["actions"] = actions
    if proprietary_fabfile_path:
        # TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
        # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
        raise Exception("Tool dependency installation using proprietary fabric scripts is not yet supported.")
    else:
        install_and_build_package_via_fabric(app, tool_dependency, actions_dict)