def diffDatabankEntries(dict_entries1, dict_entries2, verbose=True): """ Compare two Databank entries under dict format (i.e: output of getDatabankEntries) Returns None if no differences are found, or the first different key """ k = None try: verbose_compare = "if_different" if verbose else False assert len(dict_entries1) == len(dict_entries2) assert ( compare_lists( list(dict_entries1.keys()), list(dict_entries2.keys()), verbose=verbose_compare, ) == 1 ) for k in dict_entries1.keys(): v1 = dict_entries1[k] v2 = dict_entries2[k] if k in ["info", "format", "parfuncfmt", "levelsfmt"]: assert v1 == v2 elif k in ["path"]: assert ( compare_lists( [stdpath(path1) for path1 in v1], [stdpath(path2) for path2 in v2], verbose=verbose_compare, ) == 1 ) elif k in ["levels"]: assert ( compare_dict( v1, v2, compare_as_paths=list(v1.keys()), verbose=verbose_compare, ) == 1 ) else: raise ValueError("Unexpected key:", k) return None except AssertionError: if verbose: print("Key doesnt match:", k) return k
def diffDatabankEntries(dict_entries1, dict_entries2, verbose=True): ''' Compare two Databank entries under dict format (i.e: output of getDatabankEntries) Returns None if no differences are found, or the first different key ''' k = None try: verbose_compare = 'if_different' if verbose else False assert len(dict_entries1) == len(dict_entries2) assert compare_lists(list(dict_entries1.keys()), list(dict_entries2.keys()), verbose=verbose_compare) == 1 for k in dict_entries1.keys(): v1 = dict_entries1[k] v2 = dict_entries2[k] if k in ['info', 'format', 'parfuncfmt', 'levelsfmt']: assert v1 == v2 elif k in ['path']: assert compare_lists([stdpath(path1) for path1 in v1], [stdpath(path2) for path2 in v2], verbose=verbose_compare) == 1 elif k in ['levels']: assert compare_dict(v1, v2, compare_as_paths=list(v1.keys()), verbose=verbose_compare) == 1 else: raise ValueError('Unexpected key:', k) return None except AssertionError: if verbose: print('Key doesnt match:', k) return k
def check_not_deprecated( file, metadata, current_version=None, last_compatible_version=OLDEST_COMPATIBLE_VERSION, ): """Make sure cache file is not deprecated: checks that ``metadata`` is the same, and that the version under which the file was generated is valid. Parameters ---------- file: str a `` .h5`` cache file for Energy Levels metadata: dict list of variables used to create the file. If the values dont match, an error is raised. current_version: str, or ``None`` current version number. If the file was generated in a previous version a warning is raised. If ``None``, current version is read from :data:`radis.__version__`. last_backward_compatible_version: str If the file was generated in a non-compatible version, an error is raised. Default :py:data:`~radis.OLDEST_COMPATIBLE_VERSION` """ # Get attributes (metadata+version) hf = h5py.File(file, "r") try: attrs = dict(hf.attrs) except OSError: attrs = {} finally: hf.close() # Raise an error if version is not found try: file_version = attrs.pop("version") except KeyError: raise DeprecatedFileError( "File {0} has been generated in a deprecated ".format(file) + "version. Delete it to regenerate it on next run") # Get current version if current_version is None: current_version = radis.__version__ # If file version is anterior to a major change # ... Update here versions afterwhich Deprecated cache file is not safe # ... (example: a key name was changed) if parse(file_version) < parse(last_compatible_version): raise DeprecatedFileError( "File {0} has been generated in a deprecated ".format(file) + "version ({0}). Oldest compatible version is {1}. ".format( file_version, last_compatible_version) + "Delete the file to regenerate it on next run") # If file version is outdated: Warning, but no error if parse(current_version) > parse(file_version): warn( DeprecationWarning( "File {0} has been generated in ".format(file) + "a deprecated version ({0}) compared to current ({1})".format( file_version, current_version) + ". Delete it to regenerate it on next run")) out = False elif parse(current_version) == parse(file_version): out = True else: raise ValueError( "Cache file ({0}) generated with a future version ({1} > {2})? ". format(file, file_version, current_version) + "Do you own a DeLorean? Delete the file manually if you understand what happened" ) # Compare metadata metadata = _h5_compatible(metadata) out, compare_string = compare_dict(metadata, attrs, verbose=False, return_string=True) if out != 1: raise DeprecatedFileError( "Metadata in file {0} dont match ".format(file) + "expected values. See comparison below:" + "\n\tExpected\tFile\n{0}".format(compare_string)) return out
def compare_spectra(first, other, spectra_only=False, plot=True, wunit="default", verbose=True, rtol=1e-5, ignore_nan=False, ignore_outliers=False, normalize=False, **kwargs): """Compare Spectrum with another Spectrum object Parameters ---------- first: type Spectrum a Spectrum to be compared other: type Spectrum another Spectrum to compare with spectra_only: boolean, or str if ``True``, only compares spectral quantities (in the same waveunit) and not lines or conditions. If str, compare a particular quantity name. If False, compare everything (including lines and conditions and populations). Default ``False`` plot: boolean if ``True``, use plot_diff to plot all quantities for the 2 spectra and the difference between them. Default ``True``. wunit: 'nm', 'cm-1', 'default' in which wavespace to compare (and plot). If default, natural wavespace of first Spectrum is taken rtol: float relative difference to use for spectral quantities comparison ignore_nan: boolean if ``True``, nans are ignored when comparing spectral quantities ignore_outliers: boolean, or float if not False, outliers are discarded. i.e, output is determined by:: out = (~np.isclose(I, Ie, rtol=rtol, atol=0)).sum()/len(I) < ignore_outliers normalize: bool Normalize the spectra to be plotted Other Parameters ---------------- kwargs: dict arguments are forwarded to :func:`~radis.spectrum.compare.plot_diff` Returns ------- equals: boolean return True if spectra are equal (respective to tolerance defined by rtol and other input conditions) Examples -------- Compare two Spectrum objects, or specifically the transmittance:: s1.compare_with(s2) s1.compare_with(s2, 'transmittance') Note that you can also simply use `s1 == s2`, that uses :meth:`~radis.spectrum.spectrum.Spectrum.compare_with` internally:: s1 == s2 # will return True or False """ # Check inputs if not 0 <= ignore_outliers < 1: raise ValueError("ignore_outliers should be < 1, or False") if not is_spectrum(other): raise TypeError("2nd object is not a Spectrum: got class {0}".format( other.__class__)) if isinstance(spectra_only, str): # case where we compare all quantities if not spectra_only in first.get_vars(): raise ValueError( "{0} is not a spectral quantity in our Spectrum ({1})".format( spectra_only, first.get_vars())) if not spectra_only in other.get_vars(): raise ValueError( "{0} is not a spectral quantity in the other Spectrum ({1})". format(spectra_only, other.get_vars())) if verbose: # print conditions what = spectra_only if isinstance(spectra_only, str) else "all quantities" msg = "compare {0} with rtol={1}".format(what, rtol) if ignore_nan: msg += ", ignore_nan" if ignore_outliers: msg += ", ignore_outliers={0}".format(ignore_outliers) print(msg) if not plot and len(kwargs) > 0: raise ValueError("Unexpected argument: {0}".format(kwargs)) if wunit == "default": wunit = first.get_waveunit() def _compare_dataframes(df1, df2, name): """ Parameters ---------- df1, df2: pandas Dataframe lines, or vib/rovib levels dataframes name: str for error message """ # if compare_lists(df1.keys(), df2.keys(), verbose=False) != 1: # if verbose: print('... keys in {0} dont match:'.format(name)) # compare_lists(list(df1.keys()), list(df2.keys()), # verbose=True) # out = False # elif compare_lists(df1.index, df2.index, verbose=False) != 1: # if verbose: print('... index in {0} dont match:'.format(name)) # compare_lists(list(df1.index), list(df2.index), # verbose=True) # out = False # else: # out = (df1 == df2).all().all() # # return out from pandas.util.testing import assert_frame_equal try: assert_frame_equal( df1.sort_index(axis=0).sort_index(axis=1), df2.sort_index(axis=0).sort_index(axis=1), check_names=True, check_column_type= False, # solves problem in Python 2/3 dataframes (unicode/str) ) out = True except AssertionError as err: if verbose: print("Comparing ", name) print(err.args[0]) out = False return out def _compare_variables(I, Ie): """ Compare spectral quantities I and Ie """ if ignore_nan: b = ~(np.isnan(I) + np.isnan(Ie)) I = I[b] Ie = Ie[b] if ignore_outliers: out = (~np.isclose(I, Ie, rtol=rtol, atol=0)).sum() / len(I) < ignore_outliers else: out = np.allclose(I, Ie, rtol=rtol, atol=0) return bool(out) def _display_difference(q, q0): error = np.nanmax(abs(q / q0 - 1)) avgerr = np.nanmean(abs(q / q0 - 1)) print( "...", k, "don't match (up to {0:.3}% diff.,".format(error * 100) + " average {0:.3f}%)".format(avgerr * 100), ) b = True if isinstance(spectra_only, str): # compare this quantity vars = [spectra_only] else: # compare all quantities b = set(first.get_vars()) == set(other.get_vars()) if not b and verbose: print("... list of quantities dont match: {0} vs {1}".format( first.get_vars(), other.get_vars())) vars = [k for k in first.get_vars() if k in other.get_vars()] if spectra_only: # Compare spectral variables # ----------- for k in vars: w, q = first.get(k, wunit=wunit) w0, q0 = other.get(k, wunit=wunit) if len(w) != len(w0): print("Wavespaces have different length (for {0}: {1} vs {2})". format(k, len(w), len(w0))) print("We interpolate one spectrum on the other one") from scipy.interpolate import interp1d if len(q) > len(q0): f = interp1d(w, q, kind="cubic") new_q = f(w0) b1 = _compare_variables(new_q, q0) if not b1 and verbose: _display_difference(new_q, q0) else: f = interp1d(w0, q0, kind="cubic") new_q0 = f(w) b1 = _compare_variables(q, new_q0) if not b1 and verbose: _display_difference(q, new_q0) else: # no need to interpolate b1 = np.allclose(w, w0, rtol=rtol, atol=0) b1 *= _compare_variables(q, q0) if not b1 and verbose: _display_difference(q, q0) b *= b1 if plot: try: plot_diff(first, other, var=k, wunit=wunit, normalize=normalize, verbose=verbose, **kwargs) except: print("... couldn't plot {0}".format(k)) else: # Compare spectral variables # ----------- for k in vars: w, q = first.get(k, wunit=wunit) w0, q0 = other.get(k, wunit=wunit) if len(w) != len(w0): print("Wavespaces have different length (for {0}: {1} vs {2})". format(k, len(w), len(w0))) b1 = False else: b1 = np.allclose(w, w0, rtol=rtol, atol=0) b1 *= _compare_variables(q, q0) if not b1 and verbose: error = np.nanmax(abs(q / q0 - 1)) avgerr = np.nanmean(abs(q / q0 - 1)) print( "...", k, "don't match (up to {0:.3}% diff.,".format(error * 100) + " average {0:.3f}%)".format(avgerr * 100), ) b *= b1 if plot: try: plot_diff(first, other, var=k, wunit=wunit, normalize=normalize, verbose=verbose, **kwargs) except: print( "... there was an error while plotting {0}".format(k)) # Compare conditions and units # ----------- verbose_dict = "if_different" if verbose else False b1 = compare_dict(first.conditions, other.conditions, verbose=verbose_dict) == 1 b2 = compare_dict(first.cond_units, other.cond_units, verbose=verbose_dict) == 1 b3 = compare_dict(first.units, other.units, verbose=verbose_dict) == 1 if not b1 and verbose: print("... conditions don't match") if not b2 and verbose: print("... conditions units don't match") if not b3 and verbose: print("... units don't match") b *= b1 * b2 * b3 # Compare lines # ----------- if first.lines is None and other.lines is None: b4 = True elif first.lines is None: b4 = False elif other.lines is None: b4 = False else: b4 = _compare_dataframes(first.lines, other.lines, "lines") if not b4 and verbose: print("... lines dont match") b *= b4 # Compare populations # ----------- if first.populations is None and other.populations is None: b5 = True elif first.populations is None: b5 = False elif other.populations is None: b5 = False else: # Compare keys in populations b5 = True if (compare_lists(first.populations, other.populations, verbose="if_different") == 1): # same molecules, compare isotopes for molecule, isotopes in first.populations.items(): if (compare_lists( isotopes, other.populations[molecule], verbose="if_different", ) == 1): # same isotopes, compare electronic states for isotope, states in isotopes.items(): if (compare_lists( states, other.populations[molecule][isotope], verbose="if_different", ) == 1): # same electronic states, compare levels + other information for state, content in states.items(): for k, v in content.items(): if k in ["vib", "rovib"]: b5 *= _compare_dataframes( v, other.populations[molecule] [isotope][state][k], "populations of {0}({1})(iso{2})" .format( molecule, state, isotope), ) else: b5 *= (v == other.populations[ molecule][isotope][state][k]) else: b5 = False if verbose: print( "{0}(iso{1}) states are different (see above)" .format(molecule, isotope)) else: b5 = False if verbose: print("{0} isotopes are different (see above)". format(molecule)) else: b5 = False if verbose: print("Molecules are different (see above)") if not b5 and verbose: print("... populations dont match (see detail above)") b *= b5 # Compare slit # ----------- if len(first._slit) == len(other._slit) == 0: # no slit anywhere b6 = True elif len(first._slit) != len(other._slit): b6 = False if verbose: print( "A spectrum has slit function array but the other doesnt") else: ws, Is = first.get_slit() ws0, Is0 = other.get_slit() if len(ws) != len(ws0): if verbose: print("Slits have different length") b6 = False else: b6 = np.allclose(ws, ws0, rtol=rtol, atol=0) b6 *= _compare_variables(Is, Is0) if not b6 and verbose: print("Slit functions dont match") b *= b6 return bool(b)
def check_not_deprecated( file, metadata_is={}, metadata_keys_contain=[], current_version=None, last_compatible_version=OLDEST_COMPATIBLE_VERSION, ): """Make sure cache file is not deprecated: checks that ``metadata`` is the same, and that the version under which the file was generated is valid. Parameters ---------- file: str a `` .h5`` cache file for Energy Levels metadata_is: dict expected values for these variables in the file metadata. If the values dont match, a :py:func:`~radis.misc.warning.DeprecatedFileWarning` error is raised. If the file metadata contains additional keys/values, no error is raised. metadata_keys_contain: list expected list of variables in the file metadata. If the keys are not there, a :py:func:`~radis.misc.warning.DeprecatedFileWarning` error is raised. Other Parameters ---------------- current_version: str, or ``None`` current version number. If the file was generated in a previous version a warning is raised. If ``None``, current version is read from :data:`radis.__version__`. last_backward_compatible_version: str If the file was generated in a non-compatible version, an error is raised. Default :py:data:`~radis.OLDEST_COMPATIBLE_VERSION` (useful parameter to force regeneration of certain cache files after a breaking change in a new version) """ # # Get attributes (metadata+version) with pd.HDFStore(file, mode="r") as store: # Errors due to old files generated with h5py. Remove after 1.0 try: file_metadata = store.get_storer("df").attrs.metadata except AttributeError as err: if "Attribute 'metadata' does not exist in node: '/df'" in str( err): raise DeprecatedFileWarning( "Cache files metadata is read/written with pytables instead of h5py starting from radis>=0.9.28. You should regenerate your file {0}" .format(file)) from err except TypeError as err: if "cannot properly create the storer" in str(err): raise DeprecatedFileWarning( "Cache files metadata is read/written with pytables instead of h5py starting from radis>=0.9.28. You should regenerate your file {0}" .format(file)) from err # Raise an error if version is not found try: file_version = file_metadata.pop("version") except KeyError: raise DeprecatedFileWarning( "File {0} is deprecated : ".format(file) + "RADIS version missing in metadata. Delete it to regenerate it on next run" ) # Get current version if current_version is None: current_version = radis.__version__ # If file version is anterior to a major change # ... Update here versions afterwhich Deprecated cache file is not safe # ... (example: a key name was changed) if parse(file_version) < parse(last_compatible_version): raise DeprecatedFileWarning( "File {0} has been generated in a deprecated ".format(file) + "version ({0}). Oldest compatible version is {1}. ".format( file_version, last_compatible_version) + "Delete the file to regenerate it on next run") # If file version is outdated: Warning, but no error if parse(current_version) > parse(file_version): warn( DeprecationWarning( "File {0} has been generated in ".format(file) + "a deprecated version ({0}) compared to current ({1})".format( file_version, current_version) + ". Delete it to regenerate it on next run")) out = False elif parse(current_version) == parse(file_version): out = True else: raise ValueError( "Cache file ({0}) generated with a future version ({1} > {2})? ". format(file, file_version, current_version) + "Do you own a DeLorean? Delete the file manually if you understand what happened" ) # Make sure metadata keys are there: for k in metadata_keys_contain: if k not in file_metadata: raise DeprecatedFileWarning( "Metadata in file {0} doesn't contain the expected key `{1}`. " .format(file, k)) # Compare metadata values # ignore additional keys in the file attributes. metadata_is = _h5_compatible(metadata_is) file_metadata = { k: v for k, v in file_metadata.items() if k in metadata_is } out, compare_string = compare_dict( metadata_is, file_metadata, verbose=False, return_string=True, df1_str="Expected", df2_str="Got", ) if out != 1: raise DeprecatedFileWarning( "Metadata in file {0} dont match ".format(file) + "expected values. See comparison below:" + "\n\tExpected\tFile\n{0}".format(compare_string)) return out