def set_environment( app, elem, tool_shed_repository ): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> sa_session = app.model.context.current tool_dependency = None env_var_version = elem.get( 'version', '1.0' ) for env_var_elem in elem: # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_name = env_var_elem.get( 'name', None ) env_var_action = env_var_elem.get( 'action', None ) if env_var_name and env_var_action: install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None ) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', status=app.model.ToolDependency.installation_status.INSTALLING, set_status=True ) cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) if env_var_version == '1.0': # Handle setting environment variables using a fabric method. fabric_util.handle_command( app, tool_dependency, install_dir, cmd ) sa_session.refresh( tool_dependency ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) sa_session.flush() print 'Environment variable ', env_var_name, 'set in', install_dir else: raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' )
def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. sa_session = app.model.context.current tool_dependency = None # The value of package_name should match the value of the "package" type in the tool config's <requirements> tag set, but it's not required. package_name = elem.get( 'name', None ) package_version = elem.get( 'version', None ) if tool_dependencies and package_name and package_version: for package_elem in elem: if package_elem.tag == 'repository': # We have a complex repository dependency definition. tool_shed = package_elem.attrib[ 'toolshed' ] required_repository_name = package_elem.attrib[ 'name' ] required_repository_owner = package_elem.attrib[ 'owner' ] default_required_repository_changeset_revision = package_elem.attrib[ 'changeset_revision' ] required_repository = get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( app, tool_shed, required_repository_name, required_repository_owner, default_required_repository_changeset_revision ) tmp_filename = None if required_repository: required_repository_changeset_revision = required_repository.installed_changeset_revision # Define the installation directory for the required tool dependency package in the required repository. required_repository_package_install_dir = \ get_tool_dependency_install_dir( app=app, repository_name=required_repository_name, repository_owner=required_repository_owner, repository_changeset_revision=required_repository_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) # Define the this dependent repository's tool dependency installation directory that will contain the env.sh file with a path to the # required repository's installed tool dependency package. dependent_install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) # Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package. # We can get everything we need from the discovered installed required_repository. if required_repository.status in [ app.model.ToolShedRepository.installation_status.DEACTIVATED, app.model.ToolShedRepository.installation_status.INSTALLED ]: if not os.path.exists( required_repository_package_install_dir ): print 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir ) repo_files_dir = required_repository.repo_files_directory( app ) tool_dependencies_config = get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' ) if tool_dependencies_config: config_to_use = tool_dependencies_config else: message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \ ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) raise Exception( message ) else: # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. text = get_updated_changeset_revisions_from_tool_shed( app=app, tool_shed_url=tool_shed, name=required_repository_name, owner=required_repository_owner, changeset_revision=required_repository_changeset_revision ) if text: updated_changeset_revisions = listify( text ) # The list of changeset revisions is in reverse order, so the newest will be first. required_repository_changeset_revision = updated_changeset_revisions[ 0 ] # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. tmp_filename = create_temporary_tool_dependencies_config( app, tool_shed, required_repository_name, required_repository_owner, required_repository_changeset_revision ) config_to_use = tmp_filename tool_dependency, actions_dict = populate_actions_dict( app=app, dependent_install_dir=dependent_install_dir, required_install_dir=required_repository_package_install_dir, tool_shed_repository=tool_shed_repository, required_repository=required_repository, package_name=package_name, package_version=package_version, tool_dependencies_config=config_to_use ) if tmp_filename: try: os.remove( tmp_filename ) except: pass # Install and build the package via fabric. install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) raise Exception( message ) elif package_elem.tag == 'install': # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) if os.path.exists( install_dir ): print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n' tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, package_name, package_version, 'package' ) tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) sa_session.flush() else: package_install_version = package_elem.get( 'version', '1.0' ) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=app.model.ToolDependency.installation_status.INSTALLING, set_status=True ) if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. for actions_elem in package_elem: install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=package_name ) sa_session.refresh( tool_dependency ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: print package_name, 'version', package_version, 'installed in', install_dir else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) elif package_elem.tag == 'readme': # Nothing to be done. continue #elif package_elem.tag == 'proprietary_fabfile': # # TODO: This is not yet supported or functionally correct... # # Handle tool dependency installation where the repository includes one or more proprietary fabric scripts. # if not fabric_version_checked: # check_fabric_version() # fabric_version_checked = True # fabfile_name = package_elem.get( 'name', None ) # proprietary_fabfile_path = os.path.abspath( os.path.join( os.path.split( tool_dependencies_config )[ 0 ], fabfile_name ) ) # print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path return tool_dependency
def set_environment(app, elem, tool_shed_repository): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> sa_session = app.model.context.current tool_dependency = None env_var_version = elem.get('version', '1.0') for env_var_elem in elem: # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_name = env_var_elem.get('name', None) env_var_action = env_var_elem.get('action', None) if env_var_name and env_var_action: install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository. installed_changeset_revision, tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository) env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir= tool_shed_repository_install_dir) if env_var_dict: if not os.path.exists(install_dir): os.makedirs(install_dir) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', status=app.model.ToolDependency.installation_status. INSTALLING, set_status=True) cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict) if env_var_version == '1.0': # Handle setting environment variables using a fabric method. fabric_util.handle_command(app, tool_dependency, install_dir, cmd) sa_session.refresh(tool_dependency) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add(tool_dependency) sa_session.flush() print 'Environment variable ', env_var_name, 'set in', install_dir else: raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' )
def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ): """ Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency. """ action_dict = {} actions = [] for package_elem in elem: if package_elem.tag == 'install': # Create the tool_dependency record in the database. tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=app.model.ToolDependency.installation_status.INSTALLING, set_status=True ) # Get the installation method version from a tag like: <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) if package_install_version == '1.0': # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find # the <action type="set_environment"> tag. env_var_dicts = [] for actions_elem in package_elem: for action_elem in actions_elem: action_type = action_elem.get( 'type', 'shell_command' ) if action_type == 'set_environment': # <action type="set_environment"> # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> for env_elem in action_elem: if env_elem.tag == 'environment_variable': env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) if env_var_dict: if env_var_dict not in env_var_dicts: env_var_dicts.append( env_var_dict ) elif action_type == 'setup_virtualenv': # Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish. site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" ) output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr ) elif not os.path.exists( output.stdout ): log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) else: env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) ) env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) ) if env_var_dicts: if required_repository.status in [ app.model.ToolShedRepository.installation_status.INSTALLED, app.model.ToolShedRepository.installation_status.DEACTIVATED ]: # Handle the case where we have an installed required repository due to the prior_installation_required = True # setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's # tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this, # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh # file generated for the installed required repository. Each env_var_dict currently looks something like this: # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'} # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict # with the associated value in the env.sh file. new_env_var_dicts = [] env_sh_file_dir = get_tool_dependency_install_dir( app=app, repository_name=required_repository.name, repository_owner=required_repository.owner, repository_changeset_revision=required_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' ) if os.path.exists( env_sh_file_path ): for i, line in enumerate( open( env_sh_file_path, 'r' ) ): env_var_dict = env_var_dicts[ i ] action = env_var_dict.get( 'action', None ) name = env_var_dict.get( 'name', None ) value = env_var_dict.get( 'value', None ) if action and name and value: new_value = parse_env_shell_entry( action, name, value, line ) env_var_dict[ 'value' ] = new_value new_env_var_dicts.append( env_var_dict ) else: log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path ) action_dict[ 'environment_variable' ] = new_env_var_dicts else: action_dict[ 'environment_variable' ] = env_var_dicts actions.append( ( 'set_environment', action_dict ) ) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) return tool_dependency, actions return None, actions
def install_package(app, elem, tool_shed_repository, tool_dependencies=None): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. sa_session = app.model.context.current tool_dependency = None # The value of package_name should match the value of the "package" type in the tool config's <requirements> tag set, but it's not required. package_name = elem.get('name', None) package_version = elem.get('version', None) if tool_dependencies and package_name and package_version: for package_elem in elem: if package_elem.tag == 'repository': # We have a complex repository dependency definition. tool_shed = package_elem.attrib['toolshed'] required_repository_name = package_elem.attrib['name'] required_repository_owner = package_elem.attrib['owner'] default_required_repository_changeset_revision = package_elem.attrib[ 'changeset_revision'] required_repository = get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( app, tool_shed, required_repository_name, required_repository_owner, default_required_repository_changeset_revision) tmp_filename = None if required_repository: required_repository_changeset_revision = required_repository.installed_changeset_revision # Define the installation directory for the required tool dependency package in the required repository. required_repository_package_install_dir = \ get_tool_dependency_install_dir( app=app, repository_name=required_repository_name, repository_owner=required_repository_owner, repository_changeset_revision=required_repository_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) # Define the this dependent repository's tool dependency installation directory that will contain the env.sh file with a path to the # required repository's installed tool dependency package. dependent_install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository. installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) # Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package. # We can get everything we need from the discovered installed required_repository. if required_repository.status in [ app.model.ToolShedRepository.installation_status. DEACTIVATED, app.model.ToolShedRepository. installation_status.INSTALLED ]: if not os.path.exists( required_repository_package_install_dir): print 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir) repo_files_dir = required_repository.repo_files_directory( app) tool_dependencies_config = get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml') if tool_dependencies_config: config_to_use = tool_dependencies_config else: message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \ ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) raise Exception(message) else: # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. text = get_updated_changeset_revisions_from_tool_shed( app=app, tool_shed_url=tool_shed, name=required_repository_name, owner=required_repository_owner, changeset_revision= required_repository_changeset_revision) if text: updated_changeset_revisions = listify(text) # The list of changeset revisions is in reverse order, so the newest will be first. required_repository_changeset_revision = updated_changeset_revisions[ 0] # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. tmp_filename = create_temporary_tool_dependencies_config( app, tool_shed, required_repository_name, required_repository_owner, required_repository_changeset_revision) config_to_use = tmp_filename tool_dependency, actions_dict = populate_actions_dict( app=app, dependent_install_dir=dependent_install_dir, required_install_dir= required_repository_package_install_dir, tool_shed_repository=tool_shed_repository, required_repository=required_repository, package_name=package_name, package_version=package_version, tool_dependencies_config=config_to_use) if tmp_filename: try: os.remove(tmp_filename) except: pass # Install and build the package via fabric. install_and_build_package_via_fabric( app, tool_dependency, actions_dict) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) raise Exception(message) elif package_elem.tag == 'install': # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository. installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) if os.path.exists(install_dir): print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n' tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, package_name, package_version, 'package') tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add(tool_dependency) sa_session.flush() else: package_install_version = package_elem.get( 'version', '1.0') tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=app.model.ToolDependency.installation_status. INSTALLING, set_status=True) if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. for actions_elem in package_elem: install_via_fabric(app, tool_dependency, actions_elem, install_dir, package_name=package_name) sa_session.refresh(tool_dependency) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: print package_name, 'version', package_version, 'installed in', install_dir else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) elif package_elem.tag == 'readme': # Nothing to be done. continue #elif package_elem.tag == 'proprietary_fabfile': # # TODO: This is not yet supported or functionally correct... # # Handle tool dependency installation where the repository includes one or more proprietary fabric scripts. # if not fabric_version_checked: # check_fabric_version() # fabric_version_checked = True # fabfile_name = package_elem.get( 'name', None ) # proprietary_fabfile_path = os.path.abspath( os.path.join( os.path.split( tool_dependencies_config )[ 0 ], fabfile_name ) ) # print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path return tool_dependency
def handle_set_environment_entry_for_package(app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository): """ Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency. """ action_dict = {} actions = [] for package_elem in elem: if package_elem.tag == 'install': # Create the tool_dependency record in the database. tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=app.model.ToolDependency.installation_status.INSTALLING, set_status=True) # Get the installation method version from a tag like: <install version="1.0"> package_install_version = package_elem.get('version', '1.0') if package_install_version == '1.0': # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find # the <action type="set_environment"> tag. env_var_dicts = [] for actions_elem in package_elem: for action_elem in actions_elem: action_type = action_elem.get('type', 'shell_command') if action_type == 'set_environment': # <action type="set_environment"> # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> for env_elem in action_elem: if env_elem.tag == 'environment_variable': env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) if env_var_dict: if env_var_dict not in env_var_dicts: env_var_dicts.append(env_var_dict) elif action_type == 'setup_virtualenv': # Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish. site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python") output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True) if output.return_code: log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr) elif not os.path.exists(output.stdout): log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout) else: env_var_dicts.append( dict(name="PYTHONPATH", action="prepend_to", value=output.stdout)) env_var_dicts.append( dict(name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin'))) if env_var_dicts: if required_repository.status in [ app.model.ToolShedRepository.installation_status. INSTALLED, app.model.ToolShedRepository. installation_status.DEACTIVATED ]: # Handle the case where we have an installed required repository due to the prior_installation_required = True # setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's # tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this, # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh # file generated for the installed required repository. Each env_var_dict currently looks something like this: # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'} # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict # with the associated value in the env.sh file. new_env_var_dicts = [] env_sh_file_dir = get_tool_dependency_install_dir( app=app, repository_name=required_repository.name, repository_owner=required_repository.owner, repository_changeset_revision=required_repository. installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh') if os.path.exists(env_sh_file_path): for i, line in enumerate( open(env_sh_file_path, 'r')): env_var_dict = env_var_dicts[i] action = env_var_dict.get('action', None) name = env_var_dict.get('name', None) value = env_var_dict.get('value', None) if action and name and value: new_value = parse_env_shell_entry( action, name, value, line) env_var_dict['value'] = new_value new_env_var_dicts.append(env_var_dict) else: log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path) action_dict['environment_variable'] = new_env_var_dicts else: action_dict['environment_variable'] = env_var_dicts actions.append(('set_environment', action_dict)) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) return tool_dependency, actions return None, actions
def set_environment( self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install ): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> This method must also handle the sub-element tag:: <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # This function is only called for set environment actions as defined below, not within an <install version="1.0"> tool # dependency type. Here is an example of the tag set this function does handle: # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable> # </action> # Here is an example of the tag set this function does not handle: # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> env_manager = EnvManager( self.app ) tool_dependencies = [] env_var_version = elem.get( 'version', '1.0' ) tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory( self.app ) ) if elem.tag == 'environment_variable': # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> elems = [ elem ] else: # <set_environment version="1.0"> # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> # </set_environment> elems = [ env_var_elem for env_var_elem in elem ] for env_var_elem in elems: env_var_name = env_var_elem.get( 'name', None ) # The value of env_var_name must match the text value of at least 1 <requirement> tag in the # tool config's <requirements> tag set whose "type" attribute is "set_environment" (e.g., # <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_action = env_var_elem.get( 'action', None ) if env_var_name and env_var_action: # Tool dependencies of type "set_environment" always have the version attribute set to None. attr_tup = ( env_var_name, None, 'set_environment' ) if attr_tup in attr_tups_of_dependencies_for_install: install_dir = \ tool_dependency_util.get_tool_dependency_install_dir( app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None ) install_environment = InstallEnvironment( app=self.app, tool_shed_repository_install_dir=tool_shed_repository_install_dir, install_dir=install_dir ) env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem, install_environment=install_environment ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) status = self.app.install_model.ToolDependency.installation_status.INSTALLING tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency( app=self.app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', status=status, set_status=True ) if env_var_version == '1.0': # Create this tool dependency's env.sh file. env_file_builder = EnvFileBuilder( install_dir ) return_code = env_file_builder.append_line( make_executable=True, **env_var_dict ) if return_code: error_message = 'Error creating env.sh file for tool dependency %s, return_code: %s' % \ ( str( tool_dependency.name ), str( return_code ) ) log.debug( error_message ) status = self.app.install_model.ToolDependency.installation_status.ERROR tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, error_message=error_message ) else: if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR, self.app.install_model.ToolDependency.installation_status.INSTALLED ]: status = self.app.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status ) log.debug( 'Environment variable %s set in %s for tool dependency %s.' % ( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) ) else: error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' status = self.app.install_model.ToolDependency.installation_status.ERROR tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, error_message=error_message ) tool_dependencies.append( tool_dependency ) return tool_dependencies
def create_tool_dependency_with_initialized_env_sh_file( self, dependent_install_dir, tool_shed_repository, required_repository, package_name, package_version, tool_dependencies_config ): """ Create or get a tool_dependency record that is defined by the received package_name and package_version. An env.sh file will be created for the tool_dependency in the received dependent_install_dir. """ # The received required_repository refers to a tool_shed_repository record that is defined as a complex # repository dependency for this tool_dependency. The required_repository may or may not be currently # installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has # an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed, # we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh # file in either case. If the required repository ends up with an installation error, this new tool # dependency will still be fine because its containing repository will be defined as missing dependencies. tool_dependencies = [] if not os.path.exists( dependent_install_dir ): os.makedirs( dependent_install_dir ) required_tool_dependency_env_file_path = None if tool_dependencies_config: required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config ) if required_td_tree: required_td_root = required_td_tree.getroot() for required_td_elem in required_td_root: # Find the appropriate package name and version. if required_td_elem.tag == 'package': # <package name="bwa" version="0.5.9"> required_td_package_name = required_td_elem.get( 'name', None ) required_td_package_version = required_td_elem.get( 'version', None ) # Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we # find a record, we need to see if it is in an error state and if so handle it appropriately. required_tool_dependency = \ tool_dependency_util.get_tool_dependency_by_name_version_type_repository( self.app, required_repository, required_td_package_name, required_td_package_version, 'package' ) if required_td_package_name == package_name and required_td_package_version == package_version: # Get or create a database tool_dependency record with which the installed package on disk will be associated. tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency( app=self.app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=True ) # Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in # the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh # file currently exists.. required_tool_dependency_env_file_path = \ self.get_required_repository_package_env_sh_path( package_name, package_version, required_repository ) env_file_builder = EnvFileBuilder( tool_dependency.installation_directory( self.app ) ) env_file_builder.append_line( action="source", value=required_tool_dependency_env_file_path ) return_code = env_file_builder.return_code if return_code: error_message = 'Error defining env.sh file for package %s, return_code: %s' % \ ( str( package_name ), str( return_code ) ) tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR, error_message=error_message) elif required_tool_dependency is not None and required_tool_dependency.in_error_state: error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \ ( str( required_tool_dependency.name ), str( required_tool_dependency.version ), str( required_tool_dependency.status ) ) tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR, error_message=error_message) else: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.INSTALLED ) tool_dependencies.append( tool_dependency ) return tool_dependencies
def process_tag_set( self, tool_shed_repository, tool_dependency, package_elem, package_name, package_version, from_tool_migration_manager=False, tool_dependency_db_records=None ): # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. actions_elem_tuples = [] proceed_with_install = False install_dir = \ tool_dependency_util.get_tool_dependency_install_dir( app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) if os.path.exists( install_dir ): # The tool_migration_manager handles tool migration stages and the sync_database_with_file_system() # method handles two scenarios: (1) where a Galaxy file system environment related to installed # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy # database tables associated with these installed items, and (2) the Tool Shed's install and test # framework which installs repositories in 2 stages, those of type tool_dependency_definition # followed by those containing valid tools and tool functional test components. Neither of these # scenarios apply when the install manager is running. if from_tool_migration_manager: proceed_with_install = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. tool_dependency, proceed_with_install = self.sync_database_with_file_system( self.app, tool_shed_repository, package_name, package_version, install_dir, tool_dependency_type='package' ) if not proceed_with_install: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." % ( str( tool_dependency.name ), str( tool_dependency.version ) ) ) return tool_dependency, proceed_with_install, actions_elem_tuples else: proceed_with_install = True if proceed_with_install: package_install_version = package_elem.get( 'version', '1.0' ) status = self.app.install_model.ToolDependency.installation_status.INSTALLING tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency( app=self.app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=status, set_status=True ) # Get the information about the current platform in case the tool dependency definition includes tag sets # for installing compiled binaries. platform_info_dict = tool_dependency_util.get_platform_info_dict() if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem, platform_info_dict=platform_info_dict, include_after_install_actions=True ) if not actions_elem_tuples: proceed_with_install = False error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> ' error_message += 'tag set.' # Since there was an installation error, update the tool dependency status to Error. # The remove_installation_path option must be left False here. tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR, error_message=error_message) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) return tool_dependency, proceed_with_install, actions_elem_tuples