def test_replace_null(self, name_update_func, name_fetch_func):
""" Null can be replaced by non-null. """
ad = AttMap({"lone_attr": None})
assert getattr(ad, name_fetch_func)("lone_attr") is None
setter = getattr(ad, name_update_func)
non_null_value = AttMap({"was_null": "not_now"})
self._do_update(name_update_func, setter,
("lone_attr", non_null_value))
assert non_null_value == getattr(ad, name_fetch_func)("lone_attr")
class AttMapSerializationTests:
""" Tests for AttMap serialization. """
DATA_PAIRS = [('a', 1), ('b', False), ('c', range(5)),
('d', {
'A': None,
'T': []
}), ('e', AttMap({
'G': 1,
'C': [False, None]
})),
('f', [
AttMap({
"DNA": "deoxyribose",
"RNA": "ribose"
}),
AttMap({
"DNA": "thymine",
"RNA": "uracil"
})
])]
@pytest.mark.parametrize(argnames="data",
argvalues=itertools.combinations(DATA_PAIRS, 2),
ids=lambda data: " data = {}".format(str(data)))
@pytest.mark.parametrize(
argnames="data_type",
argvalues=[list, dict],
ids=lambda data_type: " data_type = {}".format(data_type))
def test_pickle_restoration(self, tmpdir, data, data_type):
""" Pickled and restored AttMap objects are identical. """
# Type the AttMap input data argument according to parameter.
data = data_type(data)
original_attrdict = AttMap(data)
filename = "attrdict-test.pkl"
# Allow either Path or raw string.
try:
dirpath = tmpdir.strpath
except AttributeError:
dirpath = tmpdir
# Serialize AttMap and write to disk.
filepath = os.path.join(dirpath, filename)
with open(filepath, 'wb') as pkl:
pickle.dump(original_attrdict, pkl)
# Validate equivalence between original and restored versions.
with open(filepath, 'rb') as pkl:
restored_attrdict = AttMap(pickle.load(pkl))
assert restored_attrdict == original_attrdict
def test_prefers_explicit_project_context(self, prj_data):
""" Explicit project data overrides any pre-stored project data. """
prj_data_modified = AttMap(copy.deepcopy(prj_data))
new_src = "src3"
new_src_val = "newpath"
assert new_src not in prj_data[DATA_SOURCES_SECTION]
prj_data_modified[DATA_SOURCES_SECTION][new_src] = new_src_val
sample_data = AttMap({
SAMPLE_NAME_COLNAME: "random-sample",
"prj": prj_data,
DATA_SOURCE_COLNAME: new_src
})
s = Sample(sample_data)
s.set_file_paths(prj_data_modified)
assert new_src_val == getattr(s, DATA_SOURCE_COLNAME)
def test_non_null(self, k, v):
""" AD is sensitive to value updates """
ad = AttMap()
assert not ad.is_null(k) and not ad.non_null(k)
ad[k] = None
assert ad.is_null(k) and not ad.non_null(k)
ad[k] = v
assert not ad.is_null(k) and ad.non_null(k)
def test_no_derived_attributes(self, prj_data, exclude_derived_attributes):
""" Passing Sample's project is equivalent to its inference. """
# Here we're disinterested in parameterization w.r.t. data source key,
# so make it constant.
src_key = self.SOURCE_KEYS[0]
# Explicitly-passed object needs to at least be an AttMap.
if exclude_derived_attributes:
prj_data.pop("derived_attributes")
sample_data = {
SAMPLE_NAME_COLNAME: "arbitrary_sample",
"prj": prj_data,
DATA_SOURCE_COLNAME: src_key
}
sample_data = AttMap(sample_data)
s = Sample(sample_data)
assert not hasattr(s, src_key)
assert src_key not in s
# Create the samples and make the calls under test.
s = Sample(sample_data)
s.set_file_paths()
# Check results.
putative_new_attr = self.DATA_SOURCES[src_key]
if exclude_derived_attributes:
# The value to which the source key maps won't have been added.
assert not hasattr(s, putative_new_attr)
assert putative_new_attr not in s
else:
# The value to which the source key maps will have been added.
assert putative_new_attr == getattr(s, DATA_SOURCE_COLNAME)
assert putative_new_attr == s[DATA_SOURCE_COLNAME]
def test_set_get_atomic(self, setter_name, getter_name, is_novel):
""" For new and existing items, validate set/get behavior. """
# Establish the AttMap for the test case.
data = dict(basic_entries())
ad = AttMap(basic_entries())
# Establish a ground truth and select name/value(s) based on
# whether or not the test case wants to test a new or existing item.
if is_novel:
item_name = "awesome_novel_attribute"
assert item_name not in ad
with pytest.raises(AttributeError):
getattr(ad, item_name)
item_values = self._TOTALLY_ARBITRARY_VALUES
else:
item_name = np.random.choice(a=list(data.keys()), size=1)[0]
item_value = data[item_name]
assert ad[item_name] == item_value
assert getattr(ad, item_name) == item_value
item_values = [item_value]
# Determine which functions to use to make the set/get calls.
setter = getattr(ad, setter_name)
getter = getattr(ad, getter_name)
# Validate set/get for each value.
for value in item_values:
setter(item_name, value)
assert getter(item_name) == value
def test_new_null(self, name_update_func, name_fetch_func):
""" When a key/item, isn't known, null is allowed. """
ad = AttMap()
setter = getattr(ad, name_update_func)
args = ("new_key", None)
self._do_update(name_update_func, setter, args)
getter = getattr(ad, name_fetch_func)
assert getter("new_key") is None
def test_construction_modes_supported(self, entries_gen,
entries_provision_type):
""" Construction wants key-value pairs; wrapping doesn't matter. """
entries_mapping = dict(entries_gen())
if entries_provision_type == "dict":
entries = entries_mapping
elif entries_provision_type == "zip":
keys, values = zip(*entries_gen())
entries = zip(keys, values)
elif entries_provision_type == "items":
entries = entries_mapping.items()
elif entries_provision_type == "list":
entries = list(entries_gen())
elif entries_provision_type == "gen":
entries = entries_gen
else:
raise ValueError(
"Unexpected entries type: {}".format(entries_provision_type))
expected = AttMap(entries_mapping)
observed = AttMap(entries)
assert expected == observed
def test_pickle_restoration(self, tmpdir, data, data_type):
""" Pickled and restored AttMap objects are identical. """
# Type the AttMap input data argument according to parameter.
data = data_type(data)
original_attrdict = AttMap(data)
filename = "attrdict-test.pkl"
# Allow either Path or raw string.
try:
dirpath = tmpdir.strpath
except AttributeError:
dirpath = tmpdir
# Serialize AttMap and write to disk.
filepath = os.path.join(dirpath, filename)
with open(filepath, 'wb') as pkl:
pickle.dump(original_attrdict, pkl)
# Validate equivalence between original and restored versions.
with open(filepath, 'rb') as pkl:
restored_attrdict = AttMap(pickle.load(pkl))
assert restored_attrdict == original_attrdict
def test_squash_existing(self, name_update_func):
""" When a value that's a mapping is assigned to existing key with
non-mapping value, the new value overwrites the old. """
ad = AttMap({"MR": 4})
assert 4 == ad.MR
assert 4 == ad["MR"]
new_value = [4, 5, 6]
args = ("MR", new_value)
setter = getattr(ad, name_update_func)
if name_update_func == "add_entries":
setter([args])
else:
setter(*args)
assert new_value == ad.MR
assert new_value == ad["MR"]
def _validate_mapping_function_implementation(entries_gen, name_comp_func):
data = dict(entries_gen())
attrdict = AttMap(data)
if __name__ == '__main__':
if name_comp_func in ["__eq__", "__ne__"]:
are_equal = getattr(attrdict, name_comp_func).__call__(data)
assert are_equal if name_comp_func == "__eq__" \
else (not are_equal)
else:
raw_dict_comp_func = getattr(data, name_comp_func)
attrdict_comp_func = getattr(attrdict, name_comp_func)
expected = raw_dict_comp_func.__call__()
observed = attrdict_comp_func.__call__()
try:
# Most comparison methods are returning iterables.
assert set(expected) == set(observed)
except TypeError:
# Could be int or other non-iterable that we're comparing.
assert expected == observed
def test_equivalence_between_implicit_and_explicit_prj(
self, prj_data, data_src_attr, src_key, explicit):
""" Passing Sample's project is equivalent to its inference. """
# Explicitly-passed object needs to at least be an AttMap.
sample_data = AttMap({
SAMPLE_NAME_COLNAME: "arbitrary_sample",
"prj": prj_data,
data_src_attr: src_key,
"derived_attributes": [data_src_attr]
})
# Create the samples and make the calls under test.
s = Sample(sample_data)
if explicit:
s.set_file_paths(sample_data.prj)
else:
s.set_file_paths()
# Check results.
expected = self.DATA_SOURCES[src_key]
observed = getattr(s, data_src_attr)
assert expected == observed
class SampleConstructorTests:
""" Basic tests of Sample's constructor """
@pytest.mark.parametrize("name_attr", [SAMPLE_NAME_COLNAME, "name"])
@pytest.mark.parametrize("fetch", [getattr, lambda s, k: s[k]])
def test_only_peppy_name(self, fetch, name_attr):
""" name and sample_name access Sample's name and work with varied syntax. """
name = "testsample"
s = Sample({SAMPLE_NAME_COLNAME: name})
assert name == fetch(s, name_attr)
@pytest.mark.parametrize("name_attr", [SAMPLE_NAME_COLNAME, "name"])
@pytest.mark.parametrize(["fetch", "exp_err"],
[(getattr, AttributeError),
(lambda s, k: s[k], KeyError)])
def test_only_snakemake_name(self, fetch, name_attr, exp_err):
""" Snakemake --> peppy <--> sample --> sample_name. """
name = "testsample"
s = Sample({SNAKEMAKE_SAMPLE_COL: name})
with pytest.raises(exp_err):
fetch(s, SNAKEMAKE_SAMPLE_COL)
assert name == fetch(s, name_attr)
@pytest.mark.parametrize("name_attr", [SAMPLE_NAME_COLNAME, "name"])
@pytest.mark.parametrize(["fetch", "exp_err"],
[(getattr, AttributeError),
(lambda s, k: s[k], KeyError)])
@pytest.mark.parametrize(["data", "expect_result"],
[({
SNAKEMAKE_SAMPLE_COL: "testsample",
SAMPLE_NAME_COLNAME: "testsample"
}, "testsample"),
({
SNAKEMAKE_SAMPLE_COL: "nameA",
SAMPLE_NAME_COLNAME: "nameB"
}, Exception)])
def test_peppy_and_snakemake_names(self, fetch, name_attr, data,
expect_result, exp_err):
""" Original peppy naming of sample name is favored; exception iff values differ. """
if isinstance(expect_result, type) and issubclass(
expect_result, Exception):
with pytest.raises(expect_result):
Sample(data)
else:
s = Sample(data)
assert expect_result == fetch(s, name_attr)
with pytest.raises(exp_err):
fetch(s, SNAKEMAKE_SAMPLE_COL)
@pytest.mark.parametrize(
["has_ref", "get_ref"],
[(lambda s: hasattr(s, PRJ_REF), lambda s: getattr(s, PRJ_REF)),
(lambda s: PRJ_REF in s, lambda s: s[PRJ_REF])])
def test_no_prj_ref(self, has_ref, get_ref):
""" Construction of a Sample without project ref --> null value """
s = Sample({SAMPLE_NAME_COLNAME: "test-sample"})
assert has_ref(s)
assert get_ref(s) is None
@pytest.mark.parametrize(
"fetch", [lambda s: getattr(s, PRJ_REF), lambda s: s[PRJ_REF]])
@pytest.mark.parametrize(["prj_ref_val", "expect"],
[(None, None), ({}, None), (AttMap(), None),
(EchoAttMap(), None), ({
"a": 1
}, PXAM({"a": 1})),
(AttMap({"b": 2}), PXAM({"b": 2})),
(PXAM({"c": 3}), PXAM({"c": 3})),
(EchoAttMap({"d": 4}), EchoAttMap({"d": 4}))])
def test_non_project_prj_ref(self, fetch, prj_ref_val, expect):
""" Project reference is null, or a PathExAttMap. """
s = Sample({SAMPLE_NAME_COLNAME: "testsample", PRJ_REF: prj_ref_val})
assert expect == fetch(s)
@pytest.mark.parametrize(
"fetch", [lambda s: getattr(s, PRJ_REF), lambda s: s[PRJ_REF]])
@pytest.mark.parametrize(["prj_ref_val", "expect"],
[(None, None), ({}, None), (AttMap(), None),
(EchoAttMap(), None), ({
"a": 1
}, PXAM({"a": 1})),
(AttMap({"b": 2}), PXAM({"b": 2})),
(PXAM({"c": 3}), PXAM({"c": 3})),
(EchoAttMap({"d": 4}), EchoAttMap({"d": 4}))])
def test_non_project_prj_ref_as_arg(self, fetch, prj_ref_val, expect):
""" Project reference must be null, or an attmap bounded above by PathExAttMap. """
s = Sample({SAMPLE_NAME_COLNAME: "testsample"}, prj=prj_ref_val)
assert expect == fetch(s)
@pytest.mark.parametrize(
"fetch",
[ #lambda s: getattr(s, PRJ_REF),
lambda s: s[PRJ_REF]
])
def test_project_prj_ref_in_data(self, proj_type, fetch, tmpdir):
""" Project is converted to PathExAttMap of sample-independent data. """
proj_data = {METADATA_KEY: {OUTDIR_KEY: tmpdir.strpath}}
prj = _get_prj(
tmpdir.join("minimal_config.yaml").strpath, proj_data, proj_type)
assert isinstance(prj, Project)
s = Sample({SAMPLE_NAME_COLNAME: "testsample", PRJ_REF: prj})
self._assert_prj_dat(proj_data, s, fetch)
@pytest.mark.parametrize(
"fetch", [lambda s: getattr(s, PRJ_REF), lambda s: s[PRJ_REF]])
def test_project_prj_ref_as_arg(self, proj_type, fetch, tmpdir):
""" Project is converted to PathExAttMap of sample-independent data. """
proj_data = {METADATA_KEY: {OUTDIR_KEY: tmpdir.strpath}}
prj = _get_prj(
tmpdir.join("minimal_config.yaml").strpath, proj_data, proj_type)
assert isinstance(prj, Project)
s = Sample({SAMPLE_NAME_COLNAME: "testsample"}, prj=prj)
self._assert_prj_dat(proj_data, s, fetch)
@pytest.mark.skip("not implemented")
def test_prj_ref_data_and_arg(self):
""" Directly-specified project is favored. """
pass
def _assert_prj_dat(self, base_data, s, fetch):
obs = fetch(s)
assert isinstance(obs, Mapping) and not isinstance(obs, Project)
exp_meta, obs_meta = base_data[METADATA_KEY], obs[METADATA_KEY]
missing = {k: v for k, v in exp_meta.items() if k not in obs_meta}
assert {} == missing
diff = {
k: (exp_meta[k], obs_meta[k])
for k in set(exp_meta.keys()) & set(obs_meta.keys())
if exp_meta[k] != obs_meta[k]
}
assert {} == diff
extra = [v for k, v in obs_meta.items() if k not in exp_meta]
assert not any(extra)
def test_text_repr_empty(func, exp):
""" Empty AttMap is correctly represented as text. """
assert exp == func(AttMap())
def test_missing_is_neither_null_nor_non_null(self, item):
""" Value of absent key is neither null nor non-null """
ad = AttMap()
assert not ad.is_null(item) and not ad.non_null(item)
def test_is_null(self, item):
""" Null-valued key/item evaluates as such. """
ad = AttMap()
ad[item] = None
assert ad.is_null(item) and not ad.non_null(item)
def _set_looper_namespace(self, pool, size):
"""
Compile a dictionary of looper/submission related settings for use in
the command templates and in submission script creation
in divvy (via adapters). Accessible via: {looper.attrname}
:param Iterable[peppy.Sample] pool: collection of sample instances
:param float size: cumulative size of the given pool
:return dict: looper/submission related settings
"""
settings = AttMap()
settings.pep_config = self.prj.config_file
settings.results_subdir = self.prj.results_folder
settings.submission_subdir = self.prj.submission_folder
settings.output_dir = self.prj.output_dir
settings.sample_output_folder = \
os.path.join(self.prj.results_folder, self._sample_lump_name(pool))
settings.job_name = self._jobname(pool)
settings.total_input_size = size
settings.log_file = \
os.path.join(self.prj.submission_folder, settings.job_name) + ".log"
settings.piface_dir = os.path.dirname(self.pl_iface.pipe_iface_file)
if hasattr(self.prj, "pipeline_config"):
# Make sure it's a file (it could be provided as null.)
pl_config_file = self.prj.pipeline_config
if pl_config_file:
if not os.path.isfile(pl_config_file):
_LOGGER.error("Pipeline config file specified "
"but not found: %s", pl_config_file)
raise IOError(pl_config_file)
_LOGGER.info("Found config file: %s", pl_config_file)
# Append arg for config file if found
settings.pipeline_config = pl_config_file
return settings
def attrdict(self, request):
""" Provide a test case with an AttMap. """
d = self.ATTR_DICT_DATA
return AttMapEcho(d) if request.getfixturevalue("return_identity") \
else AttMap(d)
def test_numeric_key(self):
""" Attribute request must be string. """
ad = AttMap({1: 'a'})
assert 'a' == ad[1]
with pytest.raises(TypeError):
getattr(ad, 1)
def looper_refgenie_populate(namespaces):
"""
A looper plugin that populates refgenie references in a PEP from
refgenie://genome/asset:tag registry paths. This can be used to convert
all refgenie references into their local paths at the looper stage, so the
final paths are passed to the workflow. This way the workflow does not
need to depend on refgenie to resolve the paths.
This is useful for example for CWL pipelines, which are built to have
paths resolved outside the workflow.
The namespaces structure required to run the plugin is:
`namespaces["pipeline"]["var_templates"]["refgenie_config"]`
:param Mapping namespaces: a nested variable namespaces dict
:return dict: sample namespace dict
:raises TypeError: if the input namespaces is not a mapping
:raises KeyError: if the namespaces mapping does not include 'pipeline'
:raises NotImplementedError: if 'var_templates' key is missing in the 'pipeline' namespace or
'refgenie_config' is missing in 'var_templates' section.
"""
if not isinstance(namespaces, Mapping):
raise TypeError("Namespaces must be a Mapping")
if "pipeline" not in namespaces:
raise KeyError(
"Namespaces do not include 'pipeline'. The job is misconfigured."
)
if (
"var_templates" in namespaces["pipeline"]
and "refgenie_config" in namespaces["pipeline"]["var_templates"]
):
rgc_path = namespaces["pipeline"]["var_templates"]["refgenie_config"]
rgc = refgenconf.RefGenConf(rgc_path)
complete_sk_dict = rgc.list_seek_keys_values()
paths_dict = {}
# This function allows you to specify tags for specific assets to use
# in the project config like:
# refgenie_asset_tags:
# genome:
# asset_name: tag_name
def get_asset_tag(genome, asset):
try:
return namespaces["project"]["refgenie"]["tag_overrides"][genome][asset]
except KeyError:
default_tag = rgc.get_default_tag(genome=genome, asset=asset)
_LOGGER.info(
f"Refgenie asset ({genome}/{asset}) tag not specified in `refgenie.tag_overrides` section. "
f"Using the default tag: {default_tag}"
)
return default_tag
except TypeError:
default_tag = rgc.get_default_tag(genome=genome, asset=asset)
_LOGGER.warn(f"tag_overrides section is malformed. Using default.")
return default_tag
# Restructure the seek key paths to make them accessible with
# {refgenie.asset_name.seek_key} in command templates
for g, gdict in complete_sk_dict.items():
_LOGGER.debug(f"Processing genome {g}")
paths_dict[g] = {}
for k, v in gdict.items():
tag = get_asset_tag(genome=g, asset=k)
# print(k,v)
try:
paths_dict[g][k] = v[tag]
except KeyError:
_LOGGER.warn(
f"Can't find tag '{tag}' for asset '{g}/{k}', as specified in your project config. Using default."
)
paths_dict[g][k] = v[rgc.get_default_tag(genome=g, asset=k)]
if "project" in namespaces and "refgenie" in namespaces["project"]:
try:
for po in namespaces["project"]["refgenie"]["path_overrides"]:
rp = prp(po["registry_path"])
_LOGGER.debug(
f"Overriding {po['registry_path']} with {po['value']}."
)
if not rp["subitem"]:
rp["subitem"] = rp["item"]
_LOGGER.debug(rp)
paths_dict[rp["namespace"]][rp["item"]][rp["subitem"]] = po["value"]
except KeyError:
_LOGGER.debug("Did not find path_overrides section")
except TypeError:
_LOGGER.warn("Warning: path_overrides is not iterable")
# print(paths_dict)
# Provide these values under the 'refgenie' namespace
namespaces["refgenie"] = AttMap(paths_dict)
return rgc.populate(namespaces)
else:
msg = """
var_templates:
refgenie_config: "$REFGENIE"
"""
_LOGGER.error(
f"refgenie_config not specified in pipeline interface. Do like so: {msg}"
)
raise NotImplementedError
def test_missing_getitem(self, missing):
attrd = AttMap()
with pytest.raises(KeyError):
attrd[missing]
def test_missing_getattr(self, missing):
attrd = AttMap()
with pytest.raises(AttributeError):
getattr(attrd, missing)
def _get_empty_attrdict(data):
ad = AttMap()
ad.add_entries(data)
return ad
def test_null_construction(self):
""" Null entries value creates empty AttMap. """
assert AttMap({}) == AttMap(None)
def test_empty_construction(self, empty_collection):
""" Empty entries container create empty AttMap. """
m = AttMap(empty_collection)
assert AttMap(None) == m
assert m != dict()