def test_module_resolver_with_mapping_specificity_rules(): # If a requirement demands a specific version, # do not map to a different version when the version # has not been specified explicitly with __test_base_path() as temp_directory: module_script = _setup_module_command( temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- blast/2.24 ''') mapping_file = os.path.join(temp_directory, "mapping") with open(mapping_file, "w") as f: f.write(''' - from: name: blast unversioned: true to: name: blast version: 2.24 ''') resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) module = resolver.resolve(ToolRequirement(name="blast", type="package")) assert module.module_name == "blast" assert module.module_version == "2.24", module.module_version # successful resolution, because Requirement does not ask for a specific version module = resolver.resolve( ToolRequirement(name="blast", version="2.22", type="package")) assert isinstance( module, NullDependency ) # NullDependency, because we don't map `version: Null` over a Requirement that asks for a specific version
def test_lmod_dependency_resolver_versionless(): with __test_base_path() as temp_directory: lmod_script = _setup_lmod_command( temp_directory, ''' /opt/apps/modulefiles: BlastPlus/2.4.0+ Infernal/1.1.2 Mothur/1.36.1 ''') resolver = LmodDependencyResolver(_SimpleDependencyManager(), lmodexec=lmod_script, versionless='true', modulepath='/path/to/modulefiles') lmod = resolver.resolve( ToolRequirement(name="Infernal", version=None, type="package")) assert lmod.module_name == "Infernal" assert lmod.module_version is None lmod = resolver.resolve( ToolRequirement(name="Mothur", version="1.36.1", type="package")) assert lmod.module_name == "Mothur" assert lmod.module_version == "1.36.1" lmod = resolver.resolve( ToolRequirement(name="BlastPlus", version="2.3", type="package")) assert lmod.module_name == "BlastPlus" assert lmod.module_version is None lmod = resolver.resolve( ToolRequirement(name="Foo", version="0.1", type="package")) assert isinstance(lmod, NullDependency)
def test_lmod_dependency_resolver_with_mapping_file(): with __test_base_path() as temp_directory: lmod_script = _setup_lmod_command( temp_directory, ''' /opt/apps/modulefiles: BlastPlus/ BlastPlus/2.2.31+ BlastPlus/2.4.0+ Infernal/ Infernal/1.1.2 Mothur/ Mothur/1.33.3 Mothur/1.36.1 Mothur/1.38.1.1 ''') mapping_file = os.path.join(temp_directory, "mapping") with open(mapping_file, "w") as f: f.write(''' - from: name: blast+ unversioned: true to: name: BlastPlus version: 2.4.0+ - from: name: Mothur version: 1.38 to: version: 1.38.1.1 ''') resolver = LmodDependencyResolver(_SimpleDependencyManager(), lmodexec=lmod_script, mapping_files=mapping_file, modulepath='/path/to/modulefiles') lmod = resolver.resolve( ToolRequirement(name="BlastPlus", version="2.2.31+", type="package")) assert lmod.module_name == "BlastPlus" assert lmod.module_version == "2.2.31+", lmod.module_version lmod = resolver.resolve(ToolRequirement(name="blast+", type="package")) assert lmod.module_name == "BlastPlus" assert lmod.module_version == "2.4.0+", lmod.module_version lmod = resolver.resolve( ToolRequirement(name="blast+", version="2.23", type="package")) assert isinstance(lmod, NullDependency) lmod = resolver.resolve( ToolRequirement(name="Infernal", version="1.2", type="package")) assert isinstance(lmod, NullDependency) lmod = resolver.resolve( ToolRequirement(name="Mothur", version="1.38", type="package")) assert lmod.module_name == "Mothur" assert lmod.module_version == "1.38.1.1", lmod.module_version
def test_against_conda_prefix_regression(): """Test that would fail if https://github.com/rtfd/readthedocs.org/issues/1902 regressed.""" base_path = mkdtemp(prefix='x' * 80) # a ridiculously long prefix try: job_dir = os.path.join(base_path, "000") dependency_manager = DependencyManager(base_path) resolver = CondaDependencyResolver( dependency_manager, auto_init=True, auto_install=True, use_path_exec= False, # For the test ensure this is always a clean install ) conda_context = resolver.conda_context assert len(list( conda_util.installed_conda_targets(conda_context))) == 0 req = ToolRequirement(name="samtools", version="0.1.16", type="package") dependency = resolver.resolve(req, job_directory=job_dir) assert dependency.shell_commands( ) is not None # install should not fail anymore installed_targets = list( conda_util.installed_conda_targets(conda_context)) assert len(installed_targets) > 0 finally: shutil.rmtree(base_path)
def get_dependencies(builder: "HasReqsHints") -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") dependencies = [] # type: List[ToolRequirement] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], software_requirement.get("packages"), ) for package in packages: version = package.get("version", None) if isinstance(version, MutableSequence): if version: version = version[0] else: version = None specs = [{"uri": s} for s in package.get("specs", [])] dependencies.append( ToolRequirement.from_dict( dict( name=cast(str, package["package"]).split("#")[-1], version=version, type="package", specs=specs, ))) return ToolRequirements.from_list(dependencies)
def test_module_resolver_with_mapping_versions(): with __test_base_path() as temp_directory: module_script = _setup_module_command( temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- blast/2.22.0-mpi blast/2.23 blast/2.24.0-mpi ''') mapping_file = os.path.join(temp_directory, "mapping") with open(mapping_file, "w") as f: f.write(''' - from: name: blast+ version: 2.24 to: name: blast version: 2.24.0-mpi - from: name: blast version: 2.22 to: version: 2.22.0-mpi ''') resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) module = resolver.resolve( ToolRequirement(name="blast+", version="2.24", type="package")) assert module.module_name == "blast" assert module.module_version == "2.24.0-mpi", module.module_version resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) module = resolver.resolve( ToolRequirement(name="blast+", version="2.23", type="package")) assert isinstance(module, NullDependency) module = resolver.resolve( ToolRequirement(name="blast", version="2.22", type="package")) assert module.module_name == "blast" assert module.module_version == "2.22.0-mpi", module.module_version
def test_docker_container_docker_cli_absent_resolve(mocker): mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None) resolver = MulledDockerContainerResolver() assert resolver.cli_available is False requirement = ToolRequirement(name="samtools", version="1.10", type="package") tool_info = ToolInfo(requirements=[requirement]) mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3') container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info) assert container_description.type == 'docker' assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3'
def test_docker_container_docker_cli_exception_resolve(mocker): mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value='/bin/docker') resolver = MulledDockerContainerResolver() assert resolver.cli_available is True requirement = ToolRequirement(name="samtools", version="1.10", type="package") tool_info = ToolInfo(requirements=[requirement]) mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3') mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.docker_cached_container_description', side_effect=CalledProcessError(1, 'bla')) container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info, install=True) assert resolver.cli_available is True assert container_description.type == 'docker' assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3'
def test_cached_singularity_container_resolver_dir_mtime_cached(mocker): mocker.patch('os.listdir', return_value=SINGULARITY_IMAGES) mocker.patch('os.path.exists', return_value=True) mocker.patch( 'galaxy.tool_util.deps.container_resolvers.mulled.safe_makedirs') mocker.patch('os.stat', return_value=mocker.Mock(st_mtime=42)) resolver = CachedMulledSingularityContainerResolver( app_info=mocker.Mock(container_image_cache_path='/'), cache_directory_cacher_type='dir_mtime') requirement = ToolRequirement(name="baz", version="2.22", type="package") tool_info = ToolInfo(requirements=[requirement]) container_description = resolver.resolve( enabled_container_types=['singularity'], tool_info=tool_info) assert container_description.type == 'singularity' assert container_description.identifier == '/singularity/mulled/baz:2.22' requirement = ToolRequirement(name="foo", version="1.0", type="package") tool_info.requirements.append(requirement) container_description = resolver.resolve( enabled_container_types=['singularity'], tool_info=tool_info) assert container_description.type == 'singularity' assert container_description.identifier == '/singularity/mulled/mulled-v2-fe8a3b846bc50d24e5df78fa0b562c43477fe9ce:9f946d13f673ab2903cb0da849ad42916d619d18-0'
def test_cached_singularity_container_resolver_uncached(mocker): mocker.patch('os.listdir', return_value=SINGULARITY_IMAGES) mocker.patch('os.path.exists', return_value=True) mocker.patch( 'galaxy.tool_util.deps.container_resolvers.mulled.safe_makedirs') resolver = CachedMulledSingularityContainerResolver(app_info=mocker.Mock( container_image_cache_path='/')) requirement = ToolRequirement(name="foo", version="1.0", type="package") tool_info = ToolInfo(requirements=[requirement]) container_description = resolver.resolve( enabled_container_types=['singularity'], tool_info=tool_info) assert container_description.type == 'singularity' assert container_description.identifier == '/singularity/mulled/foo:1.0--bar'
def test_module_dependency_resolver(): with __test_base_path() as temp_directory: module_script = _setup_module_command( temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- JAGS/3.2.0-gcc45 JAGS/3.3.0-gcc4.7.2 ProbABEL/0.1-3 ProbABEL/0.1-9e R/2.12.2 R/2.13.1 R/2.14.1 R/2.15.0 R/2.15.1 R/3.0.1(default) abokia-blast/2.0.2-130524/ompi_intel abokia-blast/2.0.2-130630/ompi_intel --------------------------- /soft/intel/modulefiles ---------------------------- advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script) module = resolver.resolve( ToolRequirement(name="R", version=None, type="package")) assert module.module_name == "R" assert module.module_version is None module = resolver.resolve( ToolRequirement(name="R", version="3.0.1", type="package")) assert module.module_name == "R" assert module.module_version == "3.0.1" module = resolver.resolve( ToolRequirement(name="R", version="3.0.4", type="package")) assert isinstance(module, NullDependency)
def test_conda_resolution(): base_path = mkdtemp() try: job_dir = os.path.join(base_path, "000") dependency_manager = DependencyManager(base_path) resolver = CondaDependencyResolver( dependency_manager, auto_init=True, auto_install=True, use_path_exec= False, # For the test ensure this is always a clean install ) req = ToolRequirement(name="samtools", version=None, type="package") dependency = resolver.resolve(req, job_directory=job_dir) assert dependency.shell_commands() is not None finally: shutil.rmtree(base_path)
def test_module_resolver_with_mapping(): with __test_base_path() as temp_directory: module_script = _setup_module_command( temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- blast/2.24 ''') mapping_file = os.path.join(temp_directory, "mapping") with open(mapping_file, "w") as f: f.write(''' - from: blast+ to: blast ''') resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) module = resolver.resolve( ToolRequirement(name="blast+", version="2.24", type="package")) assert module.module_name == "blast" assert module.module_version == "2.24", module.module_version
def test_tool_requirement_equality(): a = ToolRequirement.from_dict(REQUIREMENT_A) assert a == ToolRequirement(**REQUIREMENT_A) b = ToolRequirement(**REQUIREMENT_B) assert a != b
from collections import deque import os from .test_utils import TempDirectoryTestCase from pulsar.client.test.test_common import write_config from pulsar.client import submit_job, ClientJobDescription from pulsar.client import ClientOutputs from galaxy.tool_util.deps.dependencies import DependenciesDescription from galaxy.tool_util.deps.requirements import ToolRequirement TEST_REQUIREMENT_1 = ToolRequirement("test1", "1.0") TEST_REQUIREMENT_2 = ToolRequirement("test2", "1.0") TEST_ENV_1 = dict(name="x", value="y") class TestStager(TempDirectoryTestCase): def setUp(self): super(TestStager, self).setUp() from .test_utils import get_test_tool self.tool = get_test_tool() self.client = MockClient(self.temp_directory, self.tool) inputs = self.__setup_inputs() self.client_job_description = ClientJobDescription( tool=self.tool, command_line="run_test.exe", config_files=[], input_files=inputs, client_outputs=ClientOutputs( "/galaxy/database/working_directory/1", []), working_directory="/galaxy/database/working_directory/1", dependencies_description=DependenciesDescription(
HELP_PRIVATE_TOKEN = ("Private token used to authorize client, if the " "Pulsar server specified a private_token in app.yml " "this must match that value.") HELP_TRANSPORT = "Specify as 'curl' to use pycurl client for staging." HELP_CACHE = "Specify to test Pulsar caching during staging." HELP_TEST_ERRORS = "Specify to exercise exception handling during staging." HELP_SUPPRESS_OUTPUT = "" HELP_DISABLE_CLEANUP = ("Specify to disable cleanup after the job, this " "is useful to checking the files generated during " "the job and stored on the Pulsar server.") HELP_JOB_ID = "Submit the Pulsar job with this 'external' id." HELP_DEBUG = "Enable debug log output from Pulsar client" EXPECTED_OUTPUT = b"hello world output" EXAMPLE_UNICODE_TEXT = u'єχαмρℓє συтρυт' TEST_REQUIREMENT = ToolRequirement(name="dep1", version="1.1", type="package") TEST_DEPENDENCIES = DependenciesDescription(requirements=[TEST_REQUIREMENT]) ClientInfo = namedtuple("ClientInfo", ["client", "client_manager"]) class MockTool(object): def __init__(self, tool_dir): self.id = "client_test" self.version = "1.0" self.tool_dir = tool_dir class TestRequiredFilesObject: # it would be nice to use a RequiredFiles but that hasn't been published to PyPI yet, so just # stick to the interface here.