def test_file_size_spec_required_for_non_default_packages(
self, use_new_file_size, basic_pipe_iface_data, default_resources,
huge_resources):
""" Resource packages besides default require file size. """
# Establish the resource specification.
resource_package_data = {
"default": copy.deepcopy(default_resources),
"huge": copy.deepcopy(huge_resources)
}
# Remove file size for non-default; set it for default.
del resource_package_data["huge"]["file_size"]
if use_new_file_size:
resource_package_data["default"]["min_file_size"] = \
resource_package_data["default"].pop("file_size")
# Create the PipelineInterface.
for pipe_data in basic_pipe_iface_data.values():
pipe_data["resources"] = resource_package_data
pi = PipelineInterface(basic_pipe_iface_data)
# Attempt to select resource package should fail for each pipeline,
# regardless of the file size specification; restrict to nonnegative
# file size requests to avoid collision with ValueError that should
# arise if requesting resource package for a negative file size value.
for pipe_name in pi.pipeline_names:
with pytest.raises(KeyError):
pi.choose_resource_package(pipe_name, random.randrange(0, 10))
def test_get_pipeline_name_inferred(self):
""" Script implies pipeline name if it's not explicitly configured. """
pipeline_names = ["wgbs", "atacseq"]
for extensions in itertools.combinations(EXTENSIONS, 2):
pipelines = [
name + ext for name, ext in zip(pipeline_names, extensions)
]
pi_config_data = {pipeline: None for pipeline in pipelines}
with mock.patch("looper.models.PipelineInterface._expand_paths"):
pi = PipelineInterface(pi_config_data)
for expected_name, pipeline in zip(pipeline_names, pipelines):
assert expected_name == pi.get_pipeline_name(pipeline)
def test_get_pipeline_name_explicit(self, name_and_ext_pairs):
""" Configuration can directly specify pipeline name. """
names, extensions = zip(*name_and_ext_pairs)
pipelines = [name + ext for name, ext in name_and_ext_pairs]
pi_conf_data = {
pipeline: {
"name": name
}
for pipeline, name in zip(pipelines, names)
}
pi = PipelineInterface(pi_conf_data)
for pipeline, expected_name in zip(pipelines, names):
assert expected_name == pi.get_pipeline_name(pipeline)
def test_path_expansion(self, pipe_path, envvars, expected, config_bundles,
piface_config_bundles, pipe_iface_data):
""" User/environment variables are expanded. """
for piface_data in pipe_iface_data.values():
piface_data["path"] = pipe_path
pi = PipelineInterface(pipe_iface_data)
for _, piface_data in pi:
assert expected == piface_data["path"]
def interactive(prj_lines=PROJECT_CONFIG_LINES,
iface_lines=PIPELINE_INTERFACE_CONFIG_LINES,
merge_table_lines=MERGE_TABLE_LINES,
annotation_lines=SAMPLE_ANNOTATION_LINES,
project_kwargs=None,
logger_kwargs=None):
"""
Create Project and PipelineInterface instances from default or given data.
This is intended to provide easy access to instances of fundamental looper
object for interactive test-authorship-motivated work in an iPython
interpreter or Notebook. Test authorship is simplified if we provide
easy access to viable instances of these objects.
:param Iterable[str] prj_lines: project config lines
:param Iterable[str] iface_lines: pipeline interface config lines
:param Iterable[str] merge_table_lines: lines for a merge table file
:param Iterable[str] annotation_lines: lines for a sample annotations file
:param str | int loglevel: level at which to attend to log messages
:param dict project_kwargs: keyword arguments for Project constructor
:param dict logger_kwargs: keyword arguments for logging configuration
:param bool devmode: whether logging should be done in development mode;
this implies a more verbose level setting and a more information-rich
template for logging message formatting
:param str logfile: path to file to which to write logging messages
:return Project, PipelineInterface: one Project and one PipelineInterface,
"""
# Establish logging for interactive session.
looper_logger_kwargs = {"level": "DEBUG"}
looper_logger_kwargs.update(logger_kwargs or {})
setup_looper_logger(**looper_logger_kwargs)
# TODO: don't work with tempfiles once ctors tolerate Iterable.
dirpath = tempfile.mkdtemp()
path_conf_file = _write_temp(prj_lines,
dirpath=dirpath,
fname=P_CONFIG_FILENAME)
path_iface_file = _write_temp(iface_lines,
dirpath=dirpath,
fname="pipeline_interface.yaml")
path_merge_table_file = _write_temp(merge_table_lines,
dirpath=dirpath,
fname=MERGE_TABLE_FILENAME)
path_sample_annotation_file = _write_temp(annotation_lines,
dirpath=dirpath,
fname=ANNOTATIONS_FILENAME)
prj = Project(path_conf_file, **(project_kwargs or {}))
iface = PipelineInterface(path_iface_file)
for path in [
path_conf_file, path_iface_file, path_merge_table_file,
path_sample_annotation_file
]:
os.unlink(path)
return prj, iface
def test_no_path(self, config_bundles, piface_config_bundles,
pipe_iface_data):
""" PipelineInterface config sections need not specify path. """
pi = PipelineInterface(pipe_iface_data)
for pipe_key in self.PIPELINE_KEYS:
piface_config = pi[pipe_key]
# Specific negative test of interest.
assert "path" not in piface_config
# Positive control validation.
assert pipe_iface_data[pipe_key] == piface_config
def test_constructor_input_types(tmpdir, from_file, basic_pipe_iface_data):
""" PipelineInterface constructor handles Mapping or filepath. """
if from_file:
pipe_iface_config = tmpdir.join("pipe-iface-conf.yaml").strpath
with open(tmpdir.join("pipe-iface-conf.yaml").strpath, 'w') as f:
yaml.safe_dump(basic_pipe_iface_data, f)
else:
pipe_iface_config = basic_pipe_iface_data
pi = PipelineInterface(pipe_iface_config)
assert basic_pipe_iface_data == pi.pipe_iface_config
assert pi.pipe_iface_file == (pipe_iface_config if from_file else None)
def test_file_size_spec_not_required_for_default(self, use_new_file_size,
basic_pipe_iface_data,
default_resources,
huge_resources,
midsize_resources):
""" Default package implies minimum file size of zero. """
def clear_file_size(resource_package):
for fs_var_name in ("file_size", "min_file_size"):
if fs_var_name in resource_package:
del resource_package[fs_var_name]
# Create the resource package specification data.
resources_data = dict(
zip(["default", "midsize", "huge"], [
copy.deepcopy(data) for data in
[default_resources, midsize_resources, huge_resources]
]))
for pack_name, pack_data in resources_data.items():
# Use file size spec name as appropriate; clean default package.
if pack_name == "default":
clear_file_size(pack_data)
elif use_new_file_size:
pack_data["min_file_size"] = pack_data.pop("file_size")
# Add resource package spec data and create PipelineInterface.
pipe_iface_data = copy.deepcopy(basic_pipe_iface_data)
for pipe_data in pipe_iface_data.values():
pipe_data["resources"] = resources_data
pi = PipelineInterface(pipe_iface_data)
# We should always get default resource package for mini file.
for pipe_name, pipe_data in pi:
default_resource_package = \
pipe_data["resources"][DEFAULT_COMPUTE_RESOURCES_NAME]
clear_file_size(default_resource_package)
assert default_resource_package == \
pi.choose_resource_package(pipe_name, 0.001)
def pi_with_resources(request, basic_pipe_iface_data, resources):
""" Add resource bundle data to each config section. """
if "use_new_file_size" in request.fixturenames:
file_size_name = "min_file_size" if \
request.getfixturevalue("use_new_file_size") else "file_size"
for rp_data in resources.values():
size1 = rp_data.pop("file_size", None)
size2 = rp_data.pop("min_file_size", None)
size = size1 or size2
if size:
rp_data[file_size_name] = size
pipe_iface_config = PipelineInterface(basic_pipe_iface_data)
for pipe_data in pipe_iface_config.pipelines:
pipe_data["resources"] = resources
return pipe_iface_config
def test_relative_path(self, config_bundles, piface_config_bundles,
pipe_iface_data, pipe_path, envvars, expected,
apply_envvars):
"""
PipelineInterface construction expands pipeline path.
Environment variable(s) expand(s), but the path remains relative
if specified as such, deferring the joining with pipelines location,
which makes the path absolute, until the path is actually used.
"""
for add_path, pipe_key in zip(self.ADD_PATH, self.PIPELINE_KEYS):
if add_path:
pipe_iface_data[pipe_key]["path"] = pipe_path
pi = PipelineInterface(pipe_iface_data)
for add_path, pipe_key in zip(self.ADD_PATH, self.PIPELINE_KEYS):
if add_path:
assert expected == pi[pipe_key]["path"]
else:
assert "path" not in pi[pipe_key]