def merge_params_info(*params_infos): """ Merges parameter infos, starting from the first one and updating with each additional one. Labels (for sampled and derived) and min/max (just for derived params) are inherited from defaults (but not if one of min/max is re-defined: in that case, to avoid surprises, the other one is set to None=+/-inf) """ current_info = odict([[p, expand_info_param(v)] for p, v in params_infos[0].items() or {}]) for new_info in params_infos[1:]: if not new_info: continue for p, new_info_p in new_info.items(): if p not in current_info: current_info[p] = odict() new_info_p = expand_info_param(new_info_p) current_info[p].update(deepcopy(new_info_p)) # Account for incompatibilities: "prior" and ("value" or "derived"+bounds) incompatibilities = { _prior: [_p_value, _p_derived, "min", "max"], _p_value: [_prior, _p_ref, _p_proposal], _p_derived: [_prior, _p_drop, _p_ref, _p_proposal] } for f1, incomp in incompatibilities.items(): if f1 in new_info_p: for f2 in incomp: current_info[p].pop(f2, None) # Re-sort, so that rightmost info takes precedence *also* in the sorting new_order = chain(*[list(params) for params in params_infos[::-1]]) # The following removes duplicates maintaining order (keeps the first occurrence) new_order = list(odict.fromkeys(new_order)) current_info = odict([[p, current_info[p]] for p in new_order]) return current_info
def merge_params_info(params_infos, default_derived=True): """ Merges parameter infos, starting from the first one and updating with each additional one. Labels (for sampled and derived) and min/max (just for derived params) are inherited from defaults (but not if one of min/max is re-defined: in that case, to avoid surprises, the other one is set to None=+/-inf) """ current_info = {p: expand_info_param(v, default_derived) for p, v in params_infos[0].items() or {}} for new_info in params_infos[1:]: if not new_info: continue for p, new_info_p in new_info.items(): if p not in current_info: current_info[p] = {} new_info_p = expand_info_param(new_info_p) current_info[p].update(deepcopy(new_info_p)) # Account for incompatibilities: "prior" and ("value" or "derived"+bounds) incompatibilities = {"prior": ["value", "derived", "min", "max"], "value": ["prior", "ref", "proposal"], "derived": ["prior", "drop", "ref", "proposal"]} for f1, incomp in incompatibilities.items(): if f1 in new_info_p: for f2 in incomp: current_info[p].pop(f2, None) # type: ignore # Re-sort, so that rightmost info takes precedence *also* in the sorting new_order_sorted = chain(*params_infos[::-1]) # The following removes duplicates maintaining order (keeps the first occurrence) new_order = dict.fromkeys(new_order_sorted) current_info = {p: current_info[p] for p in new_order} return current_info
def is_equal_info(info1, info2, strict=True, print_not_log=False): """ Compares two information dictionaries. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ if print_not_log: myprint = print else: myprint = log.info myname = inspect.stack()[0][3] ignore = set([]) if strict else set( [_debug, _debug_file, _resume, _path_install, _force_reproducible]) ignore_params = (set([]) if strict else set( [_p_label, _p_renames, _p_ref, _p_proposal, "min", "max"])) if set(info1).difference(ignore) != set(info2).difference(ignore): myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % (set(info1).difference(ignore), set(info2).difference(ignore))) return False for block_name in info1: if block_name in ignore: continue block1, block2 = info1[block_name], info2[block_name] if not hasattr(block1, "keys"): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False if block_name in [_sampler, _theory]: # Internal order does NOT matter if set(block1) != set(block2): myprint(myname + ": different [%s]" % block_name) return False # Anything to ignore? for k in block1: module_folder = get_folder(k, block_name, sep=".", absolute=False) try: ignore_k = getattr( import_module(module_folder, package=_package), "ignore_at_resume", {}) except ImportError: ignore_k = {} block1k, block2k = deepcopy(block1[k]), deepcopy(block2[k]) if not strict: for kignore in ignore_k: try: block1k.pop(kignore, None) block2k.pop(kignore, None) except: pass if recursive_odict_to_dict(block1k) != recursive_odict_to_dict( block2k): myprint(myname + ": different content of [%s:%s]" % (block_name, k)) return False elif block_name in [_params, _likelihood, _prior]: # Internal order DOES matter, but just up to 1st level f = list if strict else set if f(block1) != f(block2): myprint( myname + ": different [%s] or different order of them: %r vs %r" % (block_name, list(block1), list(block2))) return False for k in block1: block1k, block2k = deepcopy(block1[k]), deepcopy(block2[k]) if block_name == _params: # Unify notation block1k = expand_info_param(block1k) block2k = expand_info_param(block2k) if not strict: for kignore in ignore_params: try: block1k.pop(kignore, None) block2k.pop(kignore, None) except: pass # Fixed params, it doesn't matter if they are saved as derived for b in [block1k, block2k]: if _p_value in b: b.pop(_p_derived, None) if (recursive_odict_to_dict(block1k) != recursive_odict_to_dict(block2k)): myprint(myname + ": different content of [%s:%s]" % (block_name, k)) return False return True
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()): """ Compares two information dictionaries, and old one versus a new one, and updates the new one for selected values of the old one. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ if print_not_log: myprint = print myprint_debug = lambda x: x else: myprint = log.info myprint_debug = log.debug myname = inspect.stack()[0][3] ignore = set() if strict else \ {_debug, _debug_file, _resume, _force, _packages_path, _test_run, _version} ignore = ignore.union(set(ignore_blocks or [])) if set(info for info in info_old if info_old[info] is not None).difference(ignore) \ != set(info for info in info_new if info_new[info] is not None).difference( ignore): myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % (set(info_old).difference(ignore), set(info_new).difference(ignore))) return False for block_name in info_old: if block_name in ignore or block_name not in info_new: continue block1 = deepcopy_where_possible(info_old[block_name]) block2 = deepcopy_where_possible(info_new[block_name]) # First, deal with root-level options (force, output, ...) if not isinstance(block1, dict): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False continue # Now let's do components and params # 1. check order (it DOES matter, but just up to 1st level) f = list if strict else set if f(block1) != f(block2): myprint(myname + ": different [%s] or different order of them: %r vs %r" % (block_name, list(block1), list(block2))) return False # 2. Gather general options to be ignored if not strict: ignore_k = set() if block_name in [kinds.theory, kinds.likelihood]: ignore_k = ignore_k.union({_input_params, _output_params}) elif block_name == _params: for param in block1: # Unify notation block1[param] = expand_info_param(block1[param]) block2[param] = expand_info_param(block2[param]) ignore_k = ignore_k.union({ partag.latex, partag.renames, partag.ref, partag.proposal, "min", "max" }) # Fixed params, it doesn't matter if they are saved as derived if partag.value in block1[param]: block1[param].pop(partag.derived, None) if partag.value in block2[param]: block2[param].pop(partag.derived, None) # Renames: order does not matter block1[param][partag.renames] = set(block1[param].get( partag.renames, [])) block2[param][partag.renames] = set(block2[param].get( partag.renames, [])) # 3. Now check component/parameters one-by-one for k in block1: if not strict: # Add component-specific options to be ignored if block_name in kinds: ignore_k_this = ignore_k.copy() if _external not in block1[k]: try: component_path = block1[k].pop(_component_path, None) \ if isinstance(block1[k], dict) else None class_name = (block1[k] or {}).get(_class_name) or k cls = get_class(class_name, block_name, component_path=component_path) ignore_k_this = ignore_k_this.union( set(getattr(cls, "_at_resume_prefer_new", {}))) except ImportError: pass # Pop ignored and kept options for j in list(ignore_k_this): block1[k].pop(j, None) block2[k].pop(j, None) if block1[k] != block2[k]: # For clarity, pop common stuff before printing to_pop = [ j for j in block1[k] if (block1[k].get(j) == block2[k].get(j)) ] [(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop] myprint(myname + ": different content of [%s:%s]" % (block_name, k) + " -- (re-run with `debug: True` for more info)") myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k])) return False return True
def is_equal_info(info_old, info_new, strict=True, print_not_log=False, ignore_blocks=()): """ Compares two information dictionaries, and old one versus a new one, and updates the new one for selected values of the old one. Set ``strict=False`` (default: ``True``) to ignore options that would not affect the statistics of a posterior sample, including order of params/priors/likelihoods. """ myprint: Callable myprint_debug: Callable if print_not_log: myprint = print myprint_debug = lambda x: x else: myprint = logger.info myprint_debug = logger.debug myname = inspect.stack()[0][3] ignorable = {"debug", "resume", "force", packages_path_input, "test", "version"} # MARKED FOR DEPRECATION IN v3.2 ignorable.add("debug_file") # END OF DEPRECATION BLOCK ignore = set() if strict else ignorable ignore = ignore.union(ignore_blocks or []) if set(info for info in info_old if info_old[info] is not None) - ignore \ != set(info for info in info_new if info_new[info] is not None) - ignore: myprint(myname + ": different blocks or options: %r (old) vs %r (new)" % ( set(info_old).difference(ignore), set(info_new).difference(ignore))) return False for block_name in info_old: if block_name in ignore or block_name not in info_new: continue block1 = deepcopy_where_possible(info_old[block_name]) block2 = deepcopy_where_possible(info_new[block_name]) # First, deal with root-level options (force, output, ...) if not isinstance(block1, dict): if block1 != block2: myprint(myname + ": different option '%s'" % block_name) return False continue # Now let's do components and params # 1. check order (it DOES matter, but just up to 1st level) f = list if strict else set if f(block1) != f(block2): myprint( myname + ": different [%s] or different order of them: %r vs %r" % ( block_name, list(block1), list(block2))) return False # 2. Gather general options to be ignored ignore_k = set() if not strict: if block_name in ["theory", "likelihood"]: ignore_k.update({"input_params", "output_params"}) elif block_name == "params": for param in block1: # Unify notation block1[param] = expand_info_param(block1[param]) block2[param] = expand_info_param(block2[param]) ignore_k.update({"latex", "renames", "ref", "proposal", "min", "max"}) # Fixed params, it doesn't matter if they are saved as derived if "value" in block1[param]: block1[param].pop("derived", None) if "value" in block2[param]: block2[param].pop("derived", None) # Renames: order does not matter block1[param]["renames"] = set(block1[param].get("renames", [])) block2[param]["renames"] = set(block2[param].get("renames", [])) # 3. Now check component/parameters one-by-one for k in block1: if not strict: # Add component-specific options to be ignored if block_name in kinds: ignore_k_this = ignore_k.union({"python_path"}) if "external" not in block1[k]: try: component_path = block1[k].pop("python_path", None) \ if isinstance(block1[k], dict) else None cls = get_component_class( k, kind=block_name, component_path=component_path, class_name=(block1[k] or {}).get("class"), logger=logger) ignore_k_this.update(set( getattr(cls, "_at_resume_prefer_new", {}))) except ImportError: pass # Pop ignored and kept options for j in ignore_k_this: block1[k].pop(j, None) block2[k].pop(j, None) if block1[k] != block2[k]: # For clarity, pop common stuff before printing to_pop = [j for j in block1[k] if (block1[k].get(j) == block2[k].get(j))] [(block1[k].pop(j, None), block2[k].pop(j, None)) for j in to_pop] myprint( myname + ": different content of [%s:%s]" % (block_name, k) + " -- (re-run with `debug: True` for more info)") myprint_debug("%r (old) vs %r (new)" % (block1[k], block2[k])) return False return True