def test_lmod_dependency_resolver_versionless():
with __test_base_path() as temp_directory:
lmod_script = _setup_lmod_command(
temp_directory, '''
/opt/apps/modulefiles:
BlastPlus/2.4.0+
Infernal/1.1.2
Mothur/1.36.1
''')
resolver = LmodDependencyResolver(_SimpleDependencyManager(),
lmodexec=lmod_script,
versionless='true',
modulepath='/path/to/modulefiles')
lmod = resolver.resolve(
ToolRequirement(name="Infernal", version=None, type="package"))
assert lmod.module_name == "Infernal"
assert lmod.module_version is None
lmod = resolver.resolve(
ToolRequirement(name="Mothur", version="1.36.1", type="package"))
assert lmod.module_name == "Mothur"
assert lmod.module_version == "1.36.1"
lmod = resolver.resolve(
ToolRequirement(name="BlastPlus", version="2.3", type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version is None
lmod = resolver.resolve(
ToolRequirement(name="Foo", version="0.1", type="package"))
assert isinstance(lmod, NullDependency)
def test_module_resolver_with_mapping_specificity_rules():
# If a requirement demands a specific version,
# do not map to a different version when the version
# has not been specified explicitly
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.24
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast
unversioned: true
to:
name: blast
version: 2.24
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(ToolRequirement(name="blast",
type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24", module.module_version # successful resolution, because Requirement does not ask for a specific version
module = resolver.resolve(
ToolRequirement(name="blast", version="2.22", type="package"))
assert isinstance(
module, NullDependency
) # NullDependency, because we don't map `version: Null` over a Requirement that asks for a specific version
def test_lmod_dependency_resolver_with_mapping_file():
with __test_base_path() as temp_directory:
lmod_script = _setup_lmod_command(
temp_directory, '''
/opt/apps/modulefiles:
BlastPlus/
BlastPlus/2.2.31+
BlastPlus/2.4.0+
Infernal/
Infernal/1.1.2
Mothur/
Mothur/1.33.3
Mothur/1.36.1
Mothur/1.38.1.1
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast+
unversioned: true
to:
name: BlastPlus
version: 2.4.0+
- from:
name: Mothur
version: 1.38
to:
version: 1.38.1.1
''')
resolver = LmodDependencyResolver(_SimpleDependencyManager(),
lmodexec=lmod_script,
mapping_files=mapping_file,
modulepath='/path/to/modulefiles')
lmod = resolver.resolve(
ToolRequirement(name="BlastPlus",
version="2.2.31+",
type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version == "2.2.31+", lmod.module_version
lmod = resolver.resolve(ToolRequirement(name="blast+", type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version == "2.4.0+", lmod.module_version
lmod = resolver.resolve(
ToolRequirement(name="blast+", version="2.23", type="package"))
assert isinstance(lmod, NullDependency)
lmod = resolver.resolve(
ToolRequirement(name="Infernal", version="1.2", type="package"))
assert isinstance(lmod, NullDependency)
lmod = resolver.resolve(
ToolRequirement(name="Mothur", version="1.38", type="package"))
assert lmod.module_name == "Mothur"
assert lmod.module_version == "1.38.1.1", lmod.module_version
def test_against_conda_prefix_regression():
"""Test that would fail if https://github.com/rtfd/readthedocs.org/issues/1902 regressed."""
base_path = mkdtemp(prefix='x' * 80) # a ridiculously long prefix
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(
conda_util.installed_conda_targets(conda_context))) == 0
req = ToolRequirement(name="samtools",
version="0.1.16",
type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands(
) is not None # install should not fail anymore
installed_targets = list(
conda_util.installed_conda_targets(conda_context))
assert len(installed_targets) > 0
finally:
shutil.rmtree(base_path)
def test_module_resolver_with_mapping_versions():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.22.0-mpi
blast/2.23
blast/2.24.0-mpi
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast+
version: 2.24
to:
name: blast
version: 2.24.0-mpi
- from:
name: blast
version: 2.22
to:
version: 2.22.0-mpi
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(
ToolRequirement(name="blast+", version="2.24", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24.0-mpi", module.module_version
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(
ToolRequirement(name="blast+", version="2.23", type="package"))
assert isinstance(module, NullDependency)
module = resolver.resolve(
ToolRequirement(name="blast", version="2.22", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.22.0-mpi", module.module_version
def test_docker_container_docker_cli_absent_resolve(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
resolver = MulledDockerContainerResolver()
assert resolver.cli_available is False
requirement = ToolRequirement(name="samtools", version="1.10", type="package")
tool_info = ToolInfo(requirements=[requirement])
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3')
container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info)
assert container_description.type == 'docker'
assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3'
def test_docker_container_docker_cli_exception_resolve(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value='/bin/docker')
resolver = MulledDockerContainerResolver()
assert resolver.cli_available is True
requirement = ToolRequirement(name="samtools", version="1.10", type="package")
tool_info = ToolInfo(requirements=[requirement])
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3')
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.docker_cached_container_description', side_effect=CalledProcessError(1, 'bla'))
container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info, install=True)
assert resolver.cli_available is True
assert container_description.type == 'docker'
assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3'
def test_cached_singularity_container_resolver_dir_mtime_cached(mocker):
mocker.patch('os.listdir', return_value=SINGULARITY_IMAGES)
mocker.patch('os.path.exists', return_value=True)
mocker.patch(
'galaxy.tool_util.deps.container_resolvers.mulled.safe_makedirs')
mocker.patch('os.stat', return_value=mocker.Mock(st_mtime=42))
resolver = CachedMulledSingularityContainerResolver(
app_info=mocker.Mock(container_image_cache_path='/'),
cache_directory_cacher_type='dir_mtime')
requirement = ToolRequirement(name="baz", version="2.22", type="package")
tool_info = ToolInfo(requirements=[requirement])
container_description = resolver.resolve(
enabled_container_types=['singularity'], tool_info=tool_info)
assert container_description.type == 'singularity'
assert container_description.identifier == '/singularity/mulled/baz:2.22'
requirement = ToolRequirement(name="foo", version="1.0", type="package")
tool_info.requirements.append(requirement)
container_description = resolver.resolve(
enabled_container_types=['singularity'], tool_info=tool_info)
assert container_description.type == 'singularity'
assert container_description.identifier == '/singularity/mulled/mulled-v2-fe8a3b846bc50d24e5df78fa0b562c43477fe9ce:9f946d13f673ab2903cb0da849ad42916d619d18-0'
def test_cached_singularity_container_resolver_uncached(mocker):
mocker.patch('os.listdir', return_value=SINGULARITY_IMAGES)
mocker.patch('os.path.exists', return_value=True)
mocker.patch(
'galaxy.tool_util.deps.container_resolvers.mulled.safe_makedirs')
resolver = CachedMulledSingularityContainerResolver(app_info=mocker.Mock(
container_image_cache_path='/'))
requirement = ToolRequirement(name="foo", version="1.0", type="package")
tool_info = ToolInfo(requirements=[requirement])
container_description = resolver.resolve(
enabled_container_types=['singularity'], tool_info=tool_info)
assert container_description.type == 'singularity'
assert container_description.identifier == '/singularity/mulled/foo:1.0--bar'
def test_module_dependency_resolver():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
JAGS/3.2.0-gcc45
JAGS/3.3.0-gcc4.7.2
ProbABEL/0.1-3
ProbABEL/0.1-9e
R/2.12.2
R/2.13.1
R/2.14.1
R/2.15.0
R/2.15.1
R/3.0.1(default)
abokia-blast/2.0.2-130524/ompi_intel
abokia-blast/2.0.2-130630/ompi_intel
--------------------------- /soft/intel/modulefiles ----------------------------
advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017
advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035
advisor/2013/update3 intel/12.0 mkl/10.2.7.041
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script)
module = resolver.resolve(
ToolRequirement(name="R", version=None, type="package"))
assert module.module_name == "R"
assert module.module_version is None
module = resolver.resolve(
ToolRequirement(name="R", version="3.0.1", type="package"))
assert module.module_name == "R"
assert module.module_version == "3.0.1"
module = resolver.resolve(
ToolRequirement(name="R", version="3.0.4", type="package"))
assert isinstance(module, NullDependency)
def test_conda_resolution():
base_path = mkdtemp()
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
req = ToolRequirement(name="samtools", version=None, type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands() is not None
finally:
shutil.rmtree(base_path)
def test_module_resolver_with_mapping():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.24
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from: blast+
to: blast
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(
ToolRequirement(name="blast+", version="2.24", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24", module.module_version
def test_tool_requirement_equality():
a = ToolRequirement.from_dict(REQUIREMENT_A)
assert a == ToolRequirement(**REQUIREMENT_A)
b = ToolRequirement(**REQUIREMENT_B)
assert a != b
from collections import deque
import os
from .test_utils import TempDirectoryTestCase
from pulsar.client.test.test_common import write_config
from pulsar.client import submit_job, ClientJobDescription
from pulsar.client import ClientOutputs
from galaxy.tool_util.deps.dependencies import DependenciesDescription
from galaxy.tool_util.deps.requirements import ToolRequirement
TEST_REQUIREMENT_1 = ToolRequirement("test1", "1.0")
TEST_REQUIREMENT_2 = ToolRequirement("test2", "1.0")
TEST_ENV_1 = dict(name="x", value="y")
class TestStager(TempDirectoryTestCase):
def setUp(self):
super(TestStager, self).setUp()
from .test_utils import get_test_tool
self.tool = get_test_tool()
self.client = MockClient(self.temp_directory, self.tool)
inputs = self.__setup_inputs()
self.client_job_description = ClientJobDescription(
tool=self.tool,
command_line="run_test.exe",
config_files=[],
input_files=inputs,
client_outputs=ClientOutputs(
"/galaxy/database/working_directory/1", []),
working_directory="/galaxy/database/working_directory/1",
dependencies_description=DependenciesDescription(
HELP_PRIVATE_TOKEN = ("Private token used to authorize client, if the "
"Pulsar server specified a private_token in app.yml "
"this must match that value.")
HELP_TRANSPORT = "Specify as 'curl' to use pycurl client for staging."
HELP_CACHE = "Specify to test Pulsar caching during staging."
HELP_TEST_ERRORS = "Specify to exercise exception handling during staging."
HELP_SUPPRESS_OUTPUT = ""
HELP_DISABLE_CLEANUP = ("Specify to disable cleanup after the job, this "
"is useful to checking the files generated during "
"the job and stored on the Pulsar server.")
HELP_JOB_ID = "Submit the Pulsar job with this 'external' id."
HELP_DEBUG = "Enable debug log output from Pulsar client"
EXPECTED_OUTPUT = b"hello world output"
EXAMPLE_UNICODE_TEXT = u'єχαмρℓє συтρυт'
TEST_REQUIREMENT = ToolRequirement(name="dep1", version="1.1", type="package")
TEST_DEPENDENCIES = DependenciesDescription(requirements=[TEST_REQUIREMENT])
ClientInfo = namedtuple("ClientInfo", ["client", "client_manager"])
class MockTool(object):
def __init__(self, tool_dir):
self.id = "client_test"
self.version = "1.0"
self.tool_dir = tool_dir
class TestRequiredFilesObject:
# it would be nice to use a RequiredFiles but that hasn't been published to PyPI yet, so just
# stick to the interface here.