def test_module_dependency_resolver():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
JAGS/3.2.0-gcc45
JAGS/3.3.0-gcc4.7.2
ProbABEL/0.1-3
ProbABEL/0.1-9e
R/2.12.2
R/2.13.1
R/2.14.1
R/2.15.0
R/2.15.1
R/3.0.1(default)
abokia-blast/2.0.2-130524/ompi_intel
abokia-blast/2.0.2-130630/ompi_intel
--------------------------- /soft/intel/modulefiles ----------------------------
advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017
advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035
advisor/2013/update3 intel/12.0 mkl/10.2.7.041
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script)
module = resolver.resolve(ToolRequirement(name="R", version=None, type="package"))
assert module.module_name == "R"
assert module.module_version is None
module = resolver.resolve(ToolRequirement(name="R", version="3.0.1", type="package"))
assert module.module_name == "R"
assert module.module_version == "3.0.1"
module = resolver.resolve(ToolRequirement(name="R", version="3.0.4", type="package"))
assert isinstance(module, NullDependency)
def test_lmod_dependency_resolver_versionless():
with __test_base_path() as temp_directory:
lmod_script = _setup_lmod_command(
temp_directory, '''
/opt/apps/modulefiles:
BlastPlus/2.4.0+
Infernal/1.1.2
Mothur/1.36.1
''')
resolver = LmodDependencyResolver(_SimpleDependencyManager(),
lmodexec=lmod_script,
versionless='true',
modulepath='/path/to/modulefiles')
lmod = resolver.resolve(
ToolRequirement(name="Infernal", version=None, type="package"))
assert lmod.module_name == "Infernal"
assert lmod.module_version is None
lmod = resolver.resolve(
ToolRequirement(name="Mothur", version="1.36.1", type="package"))
assert lmod.module_name == "Mothur"
assert lmod.module_version == "1.36.1"
lmod = resolver.resolve(
ToolRequirement(name="BlastPlus", version="2.3", type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version is None
lmod = resolver.resolve(
ToolRequirement(name="Foo", version="0.1", type="package"))
assert isinstance(lmod, NullDependency)
def test_module_resolver_with_mapping_specificity_rules():
# If a requirement demands a specific version,
# do not map to a different version when the version
# has not been specified explicitly
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.24
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast
unversioned: true
to:
name: blast
version: 2.24
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(ToolRequirement(name="blast",
type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24", module.module_version # successful resolution, because Requirement does not ask for a specific version
module = resolver.resolve(
ToolRequirement(name="blast", version="2.22", type="package"))
assert isinstance(
module, NullDependency
) # NullDependency, because we don't map `version: Null` over a Requirement that asks for a specific version
def test_lmod_dependency_resolver_with_mapping_file():
with __test_base_path() as temp_directory:
lmod_script = _setup_lmod_command(
temp_directory, '''
/opt/apps/modulefiles:
BlastPlus/
BlastPlus/2.2.31+
BlastPlus/2.4.0+
Infernal/
Infernal/1.1.2
Mothur/
Mothur/1.33.3
Mothur/1.36.1
Mothur/1.38.1.1
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast+
unversioned: true
to:
name: BlastPlus
version: 2.4.0+
- from:
name: Mothur
version: 1.38
to:
version: 1.38.1.1
''')
resolver = LmodDependencyResolver(_SimpleDependencyManager(),
lmodexec=lmod_script,
mapping_files=mapping_file,
modulepath='/path/to/modulefiles')
lmod = resolver.resolve(
ToolRequirement(name="BlastPlus",
version="2.2.31+",
type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version == "2.2.31+", lmod.module_version
lmod = resolver.resolve(ToolRequirement(name="blast+", type="package"))
assert lmod.module_name == "BlastPlus"
assert lmod.module_version == "2.4.0+", lmod.module_version
lmod = resolver.resolve(
ToolRequirement(name="blast+", version="2.23", type="package"))
assert isinstance(lmod, NullDependency)
lmod = resolver.resolve(
ToolRequirement(name="Infernal", version="1.2", type="package"))
assert isinstance(lmod, NullDependency)
lmod = resolver.resolve(
ToolRequirement(name="Mothur", version="1.38", type="package"))
assert lmod.module_name == "Mothur"
assert lmod.module_version == "1.38.1.1", lmod.module_version
def test_against_conda_prefix_regression():
"""Test that would fail if https://github.com/rtfd/readthedocs.org/issues/1902 regressed."""
base_path = mkdtemp(prefix='x' * 80) # a ridiculously long prefix
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(
conda_util.installed_conda_targets(conda_context))) == 0
req = ToolRequirement(name="samtools",
version="0.1.16",
type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands(
None) is not None # install should not fail anymore
installed_targets = list(
conda_util.installed_conda_targets(conda_context))
assert len(installed_targets) > 0
finally:
shutil.rmtree(base_path)
def test_module_resolver_with_mapping_versions():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(
temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.22.0-mpi
blast/2.23
blast/2.24.0-mpi
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from:
name: blast+
version: 2.24
to:
name: blast
version: 2.24.0-mpi
- from:
name: blast
version: 2.22
to:
version: 2.22.0-mpi
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(
ToolRequirement(name="blast+", version="2.24", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24.0-mpi", module.module_version
resolver = ModuleDependencyResolver(_SimpleDependencyManager(),
modulecmd=module_script,
mapping_files=mapping_file)
module = resolver.resolve(
ToolRequirement(name="blast+", version="2.23", type="package"))
assert isinstance(module, NullDependency)
module = resolver.resolve(
ToolRequirement(name="blast", version="2.22", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.22.0-mpi", module.module_version
def from_dict(as_dict):
if as_dict is None:
return None
requirements_dicts = as_dict.get('requirements', [])
requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts]
installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', [])
installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts)
return DependenciesDescription(
requirements=requirements,
installed_tool_dependencies=installed_tool_dependencies
)
def test_conda_resolution():
base_path = mkdtemp()
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=False, # For the test ensure this is always a clean install
)
req = ToolRequirement(name="samtools", version=None, type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands() is not None
finally:
shutil.rmtree(base_path)
def test_module_resolver_with_mapping():
with __test_base_path() as temp_directory:
module_script = _setup_module_command(temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
blast/2.24
''')
mapping_file = os.path.join(temp_directory, "mapping")
with open(mapping_file, "w") as f:
f.write('''
- from: blast+
to: blast
''')
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file)
module = resolver.resolve(ToolRequirement(name="blast+", version="2.24", type="package"))
assert module.module_name == "blast"
assert module.module_version == "2.24", module.module_version
def get_dependencies(builder): # type: (HasReqsHints) -> ToolRequirements
(software_requirement, _) = builder.get_requirement("SoftwareRequirement")
dependencies = [] # type: List[ToolRequirement]
if software_requirement and software_requirement.get("packages"):
packages = software_requirement.get("packages")
for package in packages:
version = package.get("version", None)
if isinstance(version, MutableSequence):
if version:
version = version[0]
else:
version = None
specs = [{"uri": s} for s in package.get("specs", [])]
dependencies.append(ToolRequirement.from_dict(dict(
name=package["package"].split("#")[-1],
version=version,
type="package",
specs=specs,
)))
return ToolRequirements.from_list(dependencies)
def get_dependencies(builder): # type: (HasReqsHints) -> ToolRequirements
(software_requirement, _) = builder.get_requirement("SoftwareRequirement")
dependencies = [] # type: List[ToolRequirement]
if software_requirement and software_requirement.get("packages"):
packages = software_requirement.get("packages")
for package in packages:
version = package.get("version", None)
if isinstance(version, MutableSequence):
if version:
version = version[0]
else:
version = None
specs = [{"uri": s} for s in package.get("specs", [])]
dependencies.append(ToolRequirement.from_dict(dict(
name=package["package"].split("#")[-1],
version=version,
type="package",
specs=specs,
)))
return ToolRequirements.from_list(dependencies)
makedirs(output1_extras_path)
open(join(output1_extras_path, "extra"), "w").write("EXTRA_OUTPUT_CONTENTS")
version_output.write("1.0.1")
output4_index_path.write(index_path)
finally:
output.close()
config_input.close()
output2.close()
output3.close()
version_output.close()
output4_index_path.close()
"""
EXPECTED_OUTPUT = b"hello world output"
EXAMPLE_UNICODE_TEXT = u'єχαмρℓє συтρυт'
TEST_REQUIREMENT = ToolRequirement(name="dep1", version="1.1", type="package")
TEST_DEPENDENCIES = DependenciesDescription(requirements=[TEST_REQUIREMENT])
class MockTool(object):
def __init__(self, tool_dir):
self.id = "client_test"
self.version = "1.0"
self.tool_dir = tool_dir
def run(options):
waiter = None
try:
temp_directory = tempfile.mkdtemp()
from collections import deque
import os
from .test_utils import TempDirectoryTestCase
from .test_common import write_config
from lwr.lwr_client import submit_job, ClientJobDescription
from lwr.lwr_client import ClientOutputs
from galaxy.tools.deps.dependencies import DependenciesDescription
from galaxy.tools.deps.requirements import ToolRequirement
TEST_REQUIREMENT_1 = ToolRequirement("test1", "1.0")
TEST_REQUIREMENT_2 = ToolRequirement("test2", "1.0")
TEST_ENV_1 = dict(name="x", value="y")
class TestStager(TempDirectoryTestCase):
def setUp(self):
super(TestStager, self).setUp()
from .test_utils import get_test_tool
self.tool = get_test_tool()
self.client = MockClient(self.temp_directory, self.tool)
inputs = self.__setup_inputs()
self.client_job_description = ClientJobDescription(
tool=self.tool,
command_line="run_test.exe",
config_files=[],
input_files=inputs,
client_outputs=ClientOutputs(
"/galaxy/database/working_directory/1", []),
working_directory="/galaxy/database/working_directory/1",
dependencies_description=DependenciesDescription(
def test_tool_requirement_equality():
a = ToolRequirement.from_dict(REQUIREMENT_A)
assert a == ToolRequirement(**REQUIREMENT_A)
b = ToolRequirement(**REQUIREMENT_B)
assert a != b
def test_tool_requirement_equality():
a = ToolRequirement.from_dict(REQUIREMENT_A)
assert a == ToolRequirement(**REQUIREMENT_A)
b = ToolRequirement(**REQUIREMENT_B)
assert a != b
