def merge_params_info(*params_infos):
"""
Merges parameter infos, starting from the first one
and updating with each additional one.
Labels (for sampled and derived) and min/max
(just for derived params) are inherited from defaults
(but not if one of min/max is re-defined: in that case,
to avoid surprises, the other one is set to None=+/-inf)
"""
current_info = odict([[p, expand_info_param(v)]
for p, v in params_infos[0].items() or {}])
for new_info in params_infos[1:]:
if not new_info:
continue
for p, new_info_p in new_info.items():
if p not in current_info:
current_info[p] = odict()
new_info_p = expand_info_param(new_info_p)
current_info[p].update(deepcopy(new_info_p))
# Account for incompatibilities: "prior" and ("value" or "derived"+bounds)
incompatibilities = {
_prior: [_p_value, _p_derived, "min", "max"],
_p_value: [_prior, _p_ref, _p_proposal],
_p_derived: [_prior, _p_drop, _p_ref, _p_proposal]
}
for f1, incomp in incompatibilities.items():
if f1 in new_info_p:
for f2 in incomp:
current_info[p].pop(f2, None)
# Re-sort, so that rightmost info takes precedence *also* in the sorting
new_order = chain(*[list(params) for params in params_infos[::-1]])
# The following removes duplicates maintaining order (keeps the first occurrence)
new_order = list(odict.fromkeys(new_order))
current_info = odict([[p, current_info[p]] for p in new_order])
return current_info
def merge_params_info(params_infos, default_derived=True):
"""
Merges parameter infos, starting from the first one
and updating with each additional one.
Labels (for sampled and derived) and min/max
(just for derived params) are inherited from defaults
(but not if one of min/max is re-defined: in that case,
to avoid surprises, the other one is set to None=+/-inf)
"""
current_info = {p: expand_info_param(v, default_derived) for p, v in
params_infos[0].items() or {}}
for new_info in params_infos[1:]:
if not new_info:
continue
for p, new_info_p in new_info.items():
if p not in current_info:
current_info[p] = {}
new_info_p = expand_info_param(new_info_p)
current_info[p].update(deepcopy(new_info_p))
# Account for incompatibilities: "prior" and ("value" or "derived"+bounds)
incompatibilities = {"prior": ["value", "derived", "min", "max"],
"value": ["prior", "ref", "proposal"],
"derived": ["prior", "drop", "ref", "proposal"]}
for f1, incomp in incompatibilities.items():
if f1 in new_info_p:
for f2 in incomp:
current_info[p].pop(f2, None) # type: ignore
# Re-sort, so that rightmost info takes precedence *also* in the sorting
new_order_sorted = chain(*params_infos[::-1])
# The following removes duplicates maintaining order (keeps the first occurrence)
new_order = dict.fromkeys(new_order_sorted)
current_info = {p: current_info[p] for p in new_order}
return current_info
def is_equal_info(info1, info2, strict=True, print_not_log=False):
"""
Compares two information dictionaries.
Set ``strict=False`` (default: ``True``) to ignore options that would not affect
the statistics of a posterior sample, including order of params/priors/likelihoods.
"""
if print_not_log:
myprint = print
else:
myprint = log.info
myname = inspect.stack()[0][3]
ignore = set([]) if strict else set(
[_debug, _debug_file, _resume, _path_install, _force_reproducible])
ignore_params = (set([]) if strict else set(
[_p_label, _p_renames, _p_ref, _p_proposal, "min", "max"]))
if set(info1).difference(ignore) != set(info2).difference(ignore):
myprint(myname +
": different blocks or options: %r (old) vs %r (new)" %
(set(info1).difference(ignore), set(info2).difference(ignore)))
return False
for block_name in info1:
if block_name in ignore:
continue
block1, block2 = info1[block_name], info2[block_name]
if not hasattr(block1, "keys"):
if block1 != block2:
myprint(myname + ": different option '%s'" % block_name)
return False
if block_name in [_sampler, _theory]:
# Internal order does NOT matter
if set(block1) != set(block2):
myprint(myname + ": different [%s]" % block_name)
return False
# Anything to ignore?
for k in block1:
module_folder = get_folder(k,
block_name,
sep=".",
absolute=False)
try:
ignore_k = getattr(
import_module(module_folder, package=_package),
"ignore_at_resume", {})
except ImportError:
ignore_k = {}
block1k, block2k = deepcopy(block1[k]), deepcopy(block2[k])
if not strict:
for kignore in ignore_k:
try:
block1k.pop(kignore, None)
block2k.pop(kignore, None)
except:
pass
if recursive_odict_to_dict(block1k) != recursive_odict_to_dict(
block2k):
myprint(myname + ": different content of [%s:%s]" %
(block_name, k))
return False
elif block_name in [_params, _likelihood, _prior]:
# Internal order DOES matter, but just up to 1st level
f = list if strict else set
if f(block1) != f(block2):
myprint(
myname +
": different [%s] or different order of them: %r vs %r" %
(block_name, list(block1), list(block2)))
return False
for k in block1:
block1k, block2k = deepcopy(block1[k]), deepcopy(block2[k])
if block_name == _params:
# Unify notation
block1k = expand_info_param(block1k)
block2k = expand_info_param(block2k)
if not strict:
for kignore in ignore_params:
try:
block1k.pop(kignore, None)
block2k.pop(kignore, None)
except:
pass
# Fixed params, it doesn't matter if they are saved as derived
for b in [block1k, block2k]:
if _p_value in b:
b.pop(_p_derived, None)
if (recursive_odict_to_dict(block1k) !=
recursive_odict_to_dict(block2k)):
myprint(myname + ": different content of [%s:%s]" %
(block_name, k))
return False
return True
def is_equal_info(info_old,
info_new,
strict=True,
print_not_log=False,
ignore_blocks=()):
"""
Compares two information dictionaries, and old one versus a new one, and updates the
new one for selected values of the old one.
Set ``strict=False`` (default: ``True``) to ignore options that would not affect
the statistics of a posterior sample, including order of params/priors/likelihoods.
"""
if print_not_log:
myprint = print
myprint_debug = lambda x: x
else:
myprint = log.info
myprint_debug = log.debug
myname = inspect.stack()[0][3]
ignore = set() if strict else \
{_debug, _debug_file, _resume, _force, _packages_path, _test_run, _version}
ignore = ignore.union(set(ignore_blocks or []))
if set(info for info in info_old if info_old[info] is not None).difference(ignore) \
!= set(info for info in info_new if info_new[info] is not None).difference(
ignore):
myprint(myname +
": different blocks or options: %r (old) vs %r (new)" %
(set(info_old).difference(ignore),
set(info_new).difference(ignore)))
return False
for block_name in info_old:
if block_name in ignore or block_name not in info_new:
continue
block1 = deepcopy_where_possible(info_old[block_name])
block2 = deepcopy_where_possible(info_new[block_name])
# First, deal with root-level options (force, output, ...)
if not isinstance(block1, dict):
if block1 != block2:
myprint(myname + ": different option '%s'" % block_name)
return False
continue
# Now let's do components and params
# 1. check order (it DOES matter, but just up to 1st level)
f = list if strict else set
if f(block1) != f(block2):
myprint(myname +
": different [%s] or different order of them: %r vs %r" %
(block_name, list(block1), list(block2)))
return False
# 2. Gather general options to be ignored
if not strict:
ignore_k = set()
if block_name in [kinds.theory, kinds.likelihood]:
ignore_k = ignore_k.union({_input_params, _output_params})
elif block_name == _params:
for param in block1:
# Unify notation
block1[param] = expand_info_param(block1[param])
block2[param] = expand_info_param(block2[param])
ignore_k = ignore_k.union({
partag.latex, partag.renames, partag.ref,
partag.proposal, "min", "max"
})
# Fixed params, it doesn't matter if they are saved as derived
if partag.value in block1[param]:
block1[param].pop(partag.derived, None)
if partag.value in block2[param]:
block2[param].pop(partag.derived, None)
# Renames: order does not matter
block1[param][partag.renames] = set(block1[param].get(
partag.renames, []))
block2[param][partag.renames] = set(block2[param].get(
partag.renames, []))
# 3. Now check component/parameters one-by-one
for k in block1:
if not strict:
# Add component-specific options to be ignored
if block_name in kinds:
ignore_k_this = ignore_k.copy()
if _external not in block1[k]:
try:
component_path = block1[k].pop(_component_path, None) \
if isinstance(block1[k], dict) else None
class_name = (block1[k]
or {}).get(_class_name) or k
cls = get_class(class_name,
block_name,
component_path=component_path)
ignore_k_this = ignore_k_this.union(
set(getattr(cls, "_at_resume_prefer_new", {})))
except ImportError:
pass
# Pop ignored and kept options
for j in list(ignore_k_this):
block1[k].pop(j, None)
block2[k].pop(j, None)
if block1[k] != block2[k]:
# For clarity, pop common stuff before printing
to_pop = [
j for j in block1[k]
if (block1[k].get(j) == block2[k].get(j))
]
[(block1[k].pop(j, None), block2[k].pop(j, None))
for j in to_pop]
myprint(myname + ": different content of [%s:%s]" %
(block_name, k) +
" -- (re-run with `debug: True` for more info)")
myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k]))
return False
return True
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()):
"""
Compares two information dictionaries, and old one versus a new one, and updates the
new one for selected values of the old one.
Set ``strict=False`` (default: ``True``) to ignore options that would not affect
the statistics of a posterior sample, including order of params/priors/likelihoods.
"""
myprint: Callable
myprint_debug: Callable
if print_not_log:
myprint = print
myprint_debug = lambda x: x
else:
myprint = logger.info
myprint_debug = logger.debug
myname = inspect.stack()[0][3]
ignorable = {"debug", "resume", "force", packages_path_input, "test", "version"}
# MARKED FOR DEPRECATION IN v3.2
ignorable.add("debug_file")
# END OF DEPRECATION BLOCK
ignore = set() if strict else ignorable
ignore = ignore.union(ignore_blocks or [])
if set(info for info in info_old if info_old[info] is not None) - ignore \
!= set(info for info in info_new if info_new[info] is not None) - ignore:
myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % (
set(info_old).difference(ignore), set(info_new).difference(ignore)))
return False
for block_name in info_old:
if block_name in ignore or block_name not in info_new:
continue
block1 = deepcopy_where_possible(info_old[block_name])
block2 = deepcopy_where_possible(info_new[block_name])
# First, deal with root-level options (force, output, ...)
if not isinstance(block1, dict):
if block1 != block2:
myprint(myname + ": different option '%s'" % block_name)
return False
continue
# Now let's do components and params
# 1. check order (it DOES matter, but just up to 1st level)
f = list if strict else set
if f(block1) != f(block2):
myprint(
myname + ": different [%s] or different order of them: %r vs %r" % (
block_name, list(block1), list(block2)))
return False
# 2. Gather general options to be ignored
ignore_k = set()
if not strict:
if block_name in ["theory", "likelihood"]:
ignore_k.update({"input_params", "output_params"})
elif block_name == "params":
for param in block1:
# Unify notation
block1[param] = expand_info_param(block1[param])
block2[param] = expand_info_param(block2[param])
ignore_k.update({"latex", "renames", "ref", "proposal", "min", "max"})
# Fixed params, it doesn't matter if they are saved as derived
if "value" in block1[param]:
block1[param].pop("derived", None)
if "value" in block2[param]:
block2[param].pop("derived", None)
# Renames: order does not matter
block1[param]["renames"] = set(block1[param].get("renames", []))
block2[param]["renames"] = set(block2[param].get("renames", []))
# 3. Now check component/parameters one-by-one
for k in block1:
if not strict:
# Add component-specific options to be ignored
if block_name in kinds:
ignore_k_this = ignore_k.union({"python_path"})
if "external" not in block1[k]:
try:
component_path = block1[k].pop("python_path", None) \
if isinstance(block1[k], dict) else None
cls = get_component_class(
k, kind=block_name, component_path=component_path,
class_name=(block1[k] or {}).get("class"), logger=logger)
ignore_k_this.update(set(
getattr(cls, "_at_resume_prefer_new", {})))
except ImportError:
pass
# Pop ignored and kept options
for j in ignore_k_this:
block1[k].pop(j, None)
block2[k].pop(j, None)
if block1[k] != block2[k]:
# For clarity, pop common stuff before printing
to_pop = [j for j in block1[k] if (block1[k].get(j) == block2[k].get(j))]
[(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop]
myprint(
myname + ": different content of [%s:%s]" % (block_name, k) +
" -- (re-run with `debug: True` for more info)")
myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k]))
return False
return True