def test_tool_dependencies():
# Setup directories
with __test_base_path() as base_path:
for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]:
if sub == "bin":
p = os.path.join( base_path, name, version, "bin" )
else:
p = os.path.join( base_path, name, version )
try:
makedirs( p )
except:
pass
if sub == "env.sh":
__touch( os.path.join( p, "env.sh" ) )
dm = DependencyManager( default_base_path=base_path )
dependency = dm.find_dep( "dep1", "1.0" )
assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' )
assert dependency.path == os.path.join( base_path, 'dep1', '1.0' )
assert dependency.version == "1.0"
dependency = dm.find_dep( "dep1", "2.0" )
assert dependency.script == None
assert dependency.path == os.path.join( base_path, 'dep1', '2.0' )
assert dependency.version == "2.0"
## Test default versions
symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) )
dependency = dm.find_dep( "dep1", None )
assert dependency.version == "2.0"
## Test default resolve will be fall back on default package dependency
## when using the default resolver.
dependency = dm.find_dep( "dep1", "2.1" )
assert dependency.version == "2.0" # 2.0 is defined as default_version
def test_toolshed_package_requirements():
with __test_base_path() as base_path:
test_repo = __build_test_repo('package', version=TEST_VERSION)
dm = DependencyManager( default_base_path=base_path )
package_dir = __build_ts_test_package(base_path)
dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] )
assert dependency.version == TEST_VERSION
assert dependency.script == os.path.join(package_dir, "env.sh")
def test_toolshed_tools_fallback_on_manual_dependencies():
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0")
dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] )
assert dependency.version == "1.0"
assert dependency.script == env_path
def test_toolshed_package_requirements():
with __test_base_path() as base_path:
test_repo = __build_test_repo('package', version=TEST_VERSION)
dm = DependencyManager( default_base_path=base_path )
package_dir = __build_ts_test_package(base_path)
dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] )
assert dependency.version == TEST_VERSION
assert dependency.script == os.path.join(package_dir, "env.sh")
def test_toolshed_tools_fallback_on_manual_dependencies():
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0")
dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] )
assert dependency.version == "1.0"
assert dependency.script == env_path
def test_shell_commands_built():
## Test that dependency manager builds valid shell commands for a list of
## requirements.
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
__setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" )
mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ]
commands = dm.dependency_shell_commands( mock_requirements )
__assert_foo_exported( commands )
def test_toolshed_set_enviornment_requiremetns():
with __test_base_path() as base_path:
test_repo = __build_test_repo('set_environment')
dm = DependencyManager( default_base_path=base_path )
env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET)
os.makedirs(env_settings_dir)
dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] )
assert dependency.version is None
assert dependency.script == os.path.join(env_settings_dir, "env.sh")
def test_shell_commands_built():
# Test that dependency manager builds valid shell commands for a list of
# requirements.
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
__setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" )
mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ]
commands = dm.dependency_shell_commands( mock_requirements )
__assert_foo_exported( commands )
def test_toolshed_set_enviornment_requiremetns():
with __test_base_path() as base_path:
test_repo = __build_test_repo('set_environment')
dm = DependencyManager( default_base_path=base_path )
env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET)
os.makedirs(env_settings_dir)
dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] )
assert dependency.version == None
assert dependency.script == os.path.join(env_settings_dir, "env.sh")
def test_toolshed_greater_precendence():
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
ts_package_dir = __build_ts_test_package(base_path)
gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION)
ts_env_path = os.path.join(ts_package_dir, "env.sh")
dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] )
assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used.
assert dependency.script == ts_env_path
def test_toolshed_greater_precendence():
with __test_base_path() as base_path:
dm = DependencyManager( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
ts_package_dir = __build_ts_test_package(base_path)
gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION)
ts_env_path = os.path.join(ts_package_dir, "env.sh")
dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] )
assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used.
assert dependency.script == ts_env_path
def test_against_conda_prefix_regression():
"""Test that would fail if https://github.com/rtfd/readthedocs.org/issues/1902 regressed."""
base_path = mkdtemp(prefix='x' * 80) # a ridiculously long prefix
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(
conda_util.installed_conda_targets(conda_context))) == 0
req = ToolRequirement(name="samtools",
version="0.1.16",
type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands(
None) is not None # install should not fail anymore
installed_targets = list(
conda_util.installed_conda_targets(conda_context))
assert len(installed_targets) > 0
finally:
shutil.rmtree(base_path)
def test_conda_resolution_failure():
"""This test is specifically designed to trigger https://github.com/rtfd/readthedocs.org/issues/1902
and thus it expects the install to fail. If this test fails it is a sign that the upstream
conda issue has been fixed.
"""
base_path = mkdtemp(prefix='x' * 80) # a ridiculously long prefix
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(
conda_util.installed_conda_targets(conda_context))) == 0
dependency = resolver.resolve(name="samtools",
version=None,
type="package",
job_directory=job_dir)
assert dependency.shell_commands(None) is None # install should fail
installed_targets = list(
conda_util.installed_conda_targets(conda_context))
assert len(installed_targets) == 0
finally:
shutil.rmtree(base_path)
def _build_dependency_manager_no_config(kwargs):
"""Simplified variant of build_dependency_manager from galaxy.tools.deps.
The canonical factory method requires a full Galaxy configuration object
which we do not have available in this script (an optimization).
"""
configure_logging(kwargs)
root = find_root(kwargs)
dependency_resolvers_config_file = find_path(
kwargs, "dependency_resolvers_config_file", root)
use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
parse_dependency_options(kwargs, root, dependency_resolvers_config_file)
if not use_dependencies:
dependency_manager = NullDependencyManager()
else:
dependency_manager_kwds = {
'default_base_path': tool_dependency_dir,
'conf_file': dependency_resolvers_config_file,
'app_config': kwargs,
}
if use_cached_dependency_manager:
dependency_manager_kwds[
'tool_dependency_cache_dir'] = tool_dependency_cache_dir
dependency_manager = CachedDependencyManager(
**dependency_manager_kwds)
else:
dependency_manager = DependencyManager(**dependency_manager_kwds)
return dependency_manager
def test_conda_resolution():
base_path = mkdtemp()
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=
False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(
conda_util.installed_conda_targets(conda_context))) == 0
dependency = resolver.resolve(name="samtools",
version=None,
type="package",
job_directory=job_dir)
assert dependency.shell_commands(None) is not None
installed_targets = list(
conda_util.installed_conda_targets(conda_context))
assert len(installed_targets) == 1
samtools_target = installed_targets[0]
assert samtools_target.package == "samtools"
assert samtools_target.version is None
finally:
shutil.rmtree(base_path)
def __dependency_manager(xml_content):
with __test_base_path() as base_path:
f = tempfile.NamedTemporaryFile()
f.write(xml_content)
f.flush()
dm = DependencyManager(default_base_path=base_path, conf_file=f.name)
yield dm
def __setup_dependency_manager(self, conf):
dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
resolvers_config_file = conf.get("dependency_resolvers_config_file",
"dependency_resolvers_conf.xml")
conda_config = {}
for key, value in conf.items():
if key.startswith("conda_"):
conda_config[key[len('conda_'):]] = value
self.dependency_manager = DependencyManager(dependencies_dir,
resolvers_config_file,
app_config=conda_config)
def test_tool_dependencies():
# Setup directories
with __test_base_path() as base_path:
for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]:
if sub == "bin":
p = os.path.join( base_path, name, version, "bin" )
else:
p = os.path.join( base_path, name, version )
try:
makedirs( p )
except:
pass
if sub == "env.sh":
__touch( os.path.join( p, "env.sh" ) )
dm = DependencyManager( default_base_path=base_path )
dependency = dm.find_dep( "dep1", "1.0" )
assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' )
assert dependency.path == os.path.join( base_path, 'dep1', '1.0' )
assert dependency.version == "1.0"
dependency = dm.find_dep( "dep1", "2.0" )
assert dependency.script is None
assert dependency.path == os.path.join( base_path, 'dep1', '2.0' )
assert dependency.version == "2.0"
# Test default versions
symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) )
dependency = dm.find_dep( "dep1", None )
assert dependency.version == "2.0"
# Test default resolve will be fall back on default package dependency
# when using the default resolver.
dependency = dm.find_dep( "dep1", "2.1" )
assert dependency.version == "2.0" # 2.0 is defined as default_version
def __setup_dependency_manager(self, conf):
default_tool_dependency_dir = "dependencies"
resolvers_config_file = os.path.join(self.config_dir, conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml"))
if build_dependency_manager is not None:
self.dependency_manager = build_dependency_manager(
app_config_dict=conf,
conf_file=resolvers_config_file,
default_tool_dependency_dir=default_tool_dependency_dir
)
else:
dependencies_dir = conf.get("tool_dependency_dir", default_tool_dependency_dir)
conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)
def test_conda_resolution():
base_path = mkdtemp()
try:
job_dir = os.path.join(base_path, "000")
dependency_manager = DependencyManager(base_path)
resolver = CondaDependencyResolver(
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=False, # For the test ensure this is always a clean install
)
req = ToolRequirement(name="samtools", version=None, type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands() is not None
finally:
shutil.rmtree(base_path)
def __setup_dependency_manager(self, conf):
dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
resolvers_config_file = conf.get("dependency_resolvers_config_file",
"dependency_resolvers_conf.xml")
self.dependency_manager = DependencyManager(dependencies_dir,
resolvers_config_file)
def __setup_dependency_manager(self, conf):
dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)
def __dependency_manager_for_base_path(default_base_path, conf_file=None):
return DependencyManager(default_base_path=default_base_path,
conf_file=conf_file,
app_config={"conda_auto_init": False})