def save_all_eofs_to_npzfile(self, filename: Path) -> None:
"""
Saves the complete EOF data to a numpy file.
:param filename: The filename.
"""
doys = doy_list()
eof1 = np.empty((doys.size, self.lat.size * self.long.size))
eof2 = np.empty((doys.size, self.lat.size * self.long.size))
eigenvalues = np.empty((doys.size, self.lat.size * self.long.size))
explained_variances = np.empty(
(doys.size, self.lat.size * self.long.size))
no_observations = np.empty(doys.size)
for i in range(0, doys.size):
eof = self.eof_list[i]
eof1[i, :] = eof.eof1vector
eof2[i, :] = eof.eof2vector
eigenvalues[i, :] = eof.eigenvalues
explained_variances[i, :] = eof.explained_variances
no_observations[i] = eof.no_observations
np.savez(filename,
eof1=eof1,
eof2=eof2,
explained_variances=explained_variances,
eigenvalues=eigenvalues,
no_observations=no_observations,
lat=self.lat,
long=self.long)
def restore_all_eofs_from_npzfile(filename: Path) -> EOFDataForAllDOYs:
"""
Loads all EOF data from a numpy file, which was written with :func:`EOFDataForAllDOYs.save_all_eofs_to_npzfile`.
:param filename: The filename.
:return: The EOFs for all DOYs.
"""
with np.load(filename) as data:
eof1 = data["eof1"]
eof2 = data["eof2"]
lat = data["lat"]
long = data["long"]
eigenvalues = data["eigenvalues"]
explained_variances = data["explained_variances"]
no_observations = data["no_observations"]
eofs = []
for i in range(0, doy_list().size):
eof = EOFData(lat,
long,
np.squeeze(eof1[i, :]),
np.squeeze(eof2[i, :]),
eigenvalues=np.squeeze(eigenvalues[i, :]),
explained_variances=np.squeeze(
explained_variances[i, :]),
no_observations=no_observations[i])
eofs.append(eof)
return EOFDataForAllDOYs(eofs)
def plot_eigenvalues_for_all_doys(eofs: EOFDataForAllDOYs) -> Figure:
"""
Plots the Eigenvalues for EOF1 and EOF2 for all DOYs.
:param eofs: The EOF data to plot.
:return: Handle to the figure.
"""
doygrid = doy_list()
fig = plt.figure("plot_eigenvalues_for_all_doys",
clear=True,
figsize=(6, 4),
dpi=150)
p1, = plt.plot(doygrid,
eofs.eigenvalue1_for_all_doys(),
color="blue",
label="EOF1")
p2, = plt.plot(doygrid,
eofs.eigenvalue2_for_all_doys(),
color="red",
label="EOF2")
plt.xlabel("DOY")
plt.ylabel("Eigenvalue")
plt.title("Eigenvalues")
plt.legend(handles=(p1, p2))
return fig
def plot_explained_variance_for_all_doys(
eofs: EOFDataForAllDOYs,
include_total_variance: bool = False,
include_no_observations: bool = False) -> Figure:
"""
Plots the explained variance values for EOF1 and EOF2 for all DOYs.
Comparable to Kiladis (2014), Fig. 1 (although the values there are to high by a factor of 2).
:param eofs: The EOF data to plot.
:return: Handle to the figure.
"""
doygrid = doy_list()
fig = plt.figure("plot_explained_variance_for_all_doys",
clear=True,
figsize=(6, 4),
dpi=150)
ax1 = fig.add_subplot(111)
handles = []
p1, = ax1.plot(doygrid,
eofs.explained_variance1_for_all_doys(),
color="blue",
label="EOF1")
handles.append(p1)
p2, = ax1.plot(doygrid,
eofs.explained_variance2_for_all_doys(),
color="red",
label="EOF2")
handles.append(p2)
if include_total_variance:
p3, = ax1.plot(doygrid,
eofs.total_explained_variance_for_all_doys(),
color="green",
label="Total")
handles.append(p3)
ax1.set_xlabel("DOY")
ax1.set_ylabel("Fraction of explained variance")
ax1.set_xlim((0, 366))
if include_no_observations:
ax2 = ax1.twinx()
p4, = ax2.plot(doygrid,
eofs.no_observations_for_all_doys(),
color="black",
label="Number of observations",
linestyle="--")
handles.append(p4)
ax2.set_ylabel("Number of observations")
ymin = np.min(eofs.no_observations_for_all_doys()) - np.min(
eofs.no_observations_for_all_doys()) * 0.1
ymax = np.max(eofs.no_observations_for_all_doys()) + np.max(
eofs.no_observations_for_all_doys()) * 0.1
ax2.set_ylim([ymin, ymax])
plt.title("Explained variance")
plt.legend(handles=tuple(handles))
return fig
def correct_spontaneous_sign_changes_in_eof_series(
eofs: eof.EOFDataForAllDOYs,
doy1reference: bool = False) -> eof.EOFDataForAllDOYs:
"""
Switches the signs of all pairs of EOFs (for all DOYs) if necessary, so that the signs are consistent for all DOYs.
Note that the sign of the EOFs is not uniquely defined by the PCA. Hence, the sign may jump from one DOY to another,
which can be improved using this function. As long as this step is performed before computing the PCs, it will not
change the overall result.
Generally, the sign of the EOFs for a specific DOY is changed if it differs from the sign of the EOF for the previous
DOY. The EOFs for DOY 1 are by default aligned with the original calculation by Kiladis (2014), resulting in a
an EOF series, which is totally comparable to the original Kiladis (2014) calculation. This can be switched off.
:param eofs: The EOF series for which the signs should be aligned.
:param doy1reference: If true, the EOFs of DOY 1 are aligned w.r.t to the original Kiladis (2014) calculation.
:return: The EOFs with aligned signs.
"""
switched_eofs = []
if doy1reference is True:
reference_path = Path(
os.path.dirname(
os.path.abspath(inspect.getfile(
inspect.currentframe())))) / "sign_reference"
reference_eofs = eof.load_original_eofs_for_doy(reference_path, 1)
if not reference_eofs.lat.size == eofs.lat.size \
or not reference_eofs.long.size == eofs.long.size \
or not np.all(reference_eofs.lat == eofs.lat) \
or not np.all(reference_eofs.long == eofs.long):
warnings.warn(
"References for the sign of the EOFs for DOY1 have to be interpolated to spatial grid of the"
" target EOFs. Treat results with caution.")
f1 = scipy.interpolate.interp2d(reference_eofs.long,
reference_eofs.lat,
reference_eofs.eof1map,
kind='linear')
eof1map_interpol = f1(eofs.long, eofs.lat)
f2 = scipy.interpolate.interp2d(reference_eofs.long,
reference_eofs.lat,
reference_eofs.eof2map,
kind='linear')
eof2map_interpol = f2(eofs.long, eofs.lat)
reference_eofs = eof.EOFData(eofs.lat, eofs.long, eof1map_interpol,
eof2map_interpol)
corrected_doy1 = _correct_spontaneous_sign_change_of_individual_eof(
reference_eofs, eofs.eofdata_for_doy(1))
else:
corrected_doy1 = eofs.eofdata_for_doy(1)
switched_eofs.append(corrected_doy1)
previous_eof = corrected_doy1
for doy in tools.doy_list()[1:]:
corrected_eof = _correct_spontaneous_sign_change_of_individual_eof(
previous_eof, eofs.eofdata_for_doy(doy))
switched_eofs.append(corrected_eof)
previous_eof = corrected_eof
return eof.EOFDataForAllDOYs(switched_eofs)
def interpolate_eofs_between_doys(eofs: eof.EOFDataForAllDOYs,
start_doy: int = 293,
end_doy: int = 316) -> eof.EOFDataForAllDOYs:
"""
Replaces the EOF1 and EOF2 functions between 2 DOYs by a linear interpolation between these 2 DOYs.
This should only rarely be used and has only been implemented to closely reproduce the original OMI values. There,
the EOFs have also been replaced by an interpolation according to Kiladis (2014). However, the period stated in
Kiladis (2014) from 1 November to 8 November is too short. The authors have confirmed that the right
interpolation period is from DOY 294 to DOY 315, which is used here as default value.
ATTENTION: The corresponding statistical values (e.g., the explained variances) are not changed by this routine.
So these values further on represent the original results of the PCA also for the interpolated EOFs.
:param eofs: The complete EOF series, in which the interpolation takes place.
:param start_doy: The DOY, which is used as the first point of the interpolation (i.e. start_doy + 1 is the first
element, which will be replaced by the interpolation.
:param end_doy: The DOY, which is used as the last point of the interpolation (i.e. end_doy - 1 is the last
element, which will be replaced by the interpolation.
:return: The complete EOF series with the interpolated values.
"""
doys = tools.doy_list()
start_idx = start_doy - 1
end_idx = end_doy - 1
eof_len = eofs.lat.size * eofs.long.size
eofs1 = np.empty((doys.size, eof_len))
eofs2 = np.empty((doys.size, eof_len))
# Todo: Maybe this could be solved more efficiently
# by using internal numpy functions for multidimenasional operations
for (idx, doy) in enumerate(doys):
eofs1[idx, :] = eofs.eof1vector_for_doy(doy)
eofs2[idx, :] = eofs.eof2vector_for_doy(doy)
for i in range(0, eof_len):
eofs1[start_idx + 1:end_idx - 1,
i] = np.interp(doys[start_idx + 1:end_idx - 1],
[doys[start_idx], doys[end_idx]],
[eofs1[start_idx, i], eofs1[end_idx, i]])
eofs2[start_idx + 1:end_idx - 1,
i] = np.interp(doys[start_idx + 1:end_idx - 1],
[doys[start_idx], doys[end_idx]],
[eofs2[start_idx, i], eofs2[end_idx, i]])
interpolated_eofs = []
for (idx, doy) in enumerate(doys):
orig_eof = eofs.eofdata_for_doy(doy)
interpolated_eofs.append(
eof.EOFData(orig_eof.lat,
orig_eof.long,
np.squeeze(eofs1[idx, :]),
np.squeeze(eofs2[idx, :]),
explained_variances=orig_eof.explained_variances,
eigenvalues=orig_eof.eigenvalues,
no_observations=orig_eof.no_observations))
return eof.EOFDataForAllDOYs(interpolated_eofs)
def eigenvalue2_for_all_doys(self):
"""
Returns a vector with 366 elements containing the eigenvalues of EOF2 for each DOY.
:return: The eigenvalue vector.
"""
doys = doy_list()
result = []
for doy in doys:
result.append(self.eofdata_for_doy(doy).eigenvalue_eof2)
return np.array(result)
def explained_variance2_for_all_doys(self):
"""
Returns a vector with 366 elements containing the explained variance of EOF2 for each DOY.
:return: The variance vector.
"""
doys = doy_list()
result = []
for doy in doys:
result.append(self.eofdata_for_doy(doy).explained_variance_eof2)
return np.array(result)
def total_explained_variance_for_all_doys(self):
"""
Returns a vector with 366 elements containing for each DOY the sum of the explained variance over all EOFs.
:return: The variance vector. Should by close to 1 for each DOY if computation was successful.
"""
doys = doy_list()
result = []
for doy in doys:
result.append(self.eofdata_for_doy(doy).sum_of_explained_variances)
return np.array(result)
def plot_correlation_for_eofs_all_doys(
recalc_eof: eof.EOFDataForAllDOYs,
orig_eof: eof.EOFDataForAllDOYs,
exclude_doy366: bool = False,
do_print: bool = False,
full_value_range: bool = True) -> Figure:
"""
Plots the correlations between calculated EOFs and reference EOFs.
Correlations will be shown for each DOY (DOY on the abscissa) and with one line each for EOF1 and EOF2.
:param recalc_eof: The EOFs to validate.
:param orig_eof: The reference EOFs.
:param exclude_doy366: If False, DOY 366 will be included in the plot (sometimes worse correlation depending on the
leap year treatment mode).
:param do_print: If True, some characteristic values will we printed to the console.
:param full_value_range: If True, the ordinate spans the range from 0 to 1 instead of the used value range only.
:return: A handle to the figure.
"""
doys = tools.doy_list()
if exclude_doy366:
doys = doys[:-1]
xlim = (0, doys[-1])
(corr_1, diff_mean_1, diff_std_1, diff_abs_percent68_1, diff_abs_percent95_1, diff_abs_percent99_1)\
= calc_comparison_stats_for_eofs_all_doys(orig_eof, recalc_eof, exclude_doy366=exclude_doy366, eof_number=1,
percentage=False, do_print=do_print)
(corr_2, diff_mean_2, diff_std_2, diff_abs_percent68_2, diff_abs_percent95_2, diff_abs_percent99_2)\
= calc_comparison_stats_for_eofs_all_doys(orig_eof, recalc_eof, exclude_doy366=exclude_doy366, eof_number=2,
percentage=False, do_print=do_print)
fig_id = "plot_correlation_for_eofs_all_doys"
fig, axs = plt.subplots(1,
1,
num=fig_id,
clear=True,
figsize=(6, 4.5),
dpi=150)
plt.subplots_adjust(hspace=0.3)
fig.suptitle("Correlation of EOFs for all DOYs")
ax = axs
p11, = ax.plot(doys, corr_1, label="EOF1", color="blue")
p12, = ax.plot(doys, corr_2, label="EOF2", color="green")
ax.set_xlim(xlim)
ax.legend(handles=[p11, p12])
ax.set_xlabel("Day of year")
ax.set_ylabel("Correlation coefficient")
if full_value_range:
ax.set_ylim((0, 1.1))
return fig
def no_observations_for_all_doys(self):
"""
Returns a vector with 366 elements containing for each DOY the number of observations that went into the
computation of the EOFs.
:return: The number of observations vector.
"""
doys = doy_list()
result = []
for doy in doys:
result.append(self.eofdata_for_doy(doy).no_observations)
return np.array(result)
def load_all_eofs_from_directory(dirname: Path) -> EOFDataForAllDOYs:
"""
Loads the EOF functions (created with the function :func:`EOFDataForAllDOYs.save_all_eofs_to_dir`)
for all DOYs from the given directory
:param dirname: The directory in which the files are stored.
:return: The EOFs for all DOYs.
"""
eofs = []
for doy in doy_list():
filename = dirname / Path("eof%s.txt" % format(doy, '03'))
eof = load_single_eofs_from_txt_file(filename)
eofs.append(eof)
return EOFDataForAllDOYs(eofs)
def save_all_eofs_to_dir(self, dirname: Path, create_dir=True) -> None:
"""
Saves the EOF1 and EOF2 functions for each of the DOYs in the given directory.
For each DOY, one text file will be created, which contains both EOF functions.
Note that the text files do not contain the eigenvalues and explained variance values. To save also those
values, use the function :func:`save_all_eofs_to_npzfile`.
:param dirname: The directory, where the files will be saved into.
:param create_dir: If True, the directory (and parent directories) will be created, if not existing.
"""
if not dirname.exists() and create_dir:
dirname.mkdir(parents=True, exist_ok=False)
for doy in doy_list():
filename = dirname / Path("eof%s.txt" % format(doy, '03'))
self.eofdata_for_doy(doy).save_eofs_to_txt_file(filename)
def test_EOFDataForAllDOYs_alldoy_getfunctions():
doys = tools.doy_list()
lat = np.array([-10., 0., 10.])
long = np.array([0., 5.])
explained_variances = np.array([np.arange(1, doys.size + 1, 1) + 111,
np.arange(1, doys.size + 1, 1) + 222,
np.arange(1, doys.size + 1, 1) + 333,
np.arange(1, doys.size + 1, 1) + 444,
np.arange(1, doys.size + 1, 1) + 555,
np.arange(1, doys.size + 1, 1) + 666])
eigenvalues = np.array([np.arange(1, doys.size + 1, 1) + 1111,
np.arange(1, doys.size + 1, 1) + 2222,
np.arange(1, doys.size + 1, 1) + 3333,
np.arange(1, doys.size + 1, 1) + 4444,
np.arange(1, doys.size + 1, 1) + 5555,
np.arange(1, doys.size + 1, 1) + 6666])
no_obs = doys * 5
eofs = []
for doy in doys:
eof1 = np.array([1, 2, 3, 4, 5, 6]) * doy
eof2 = np.array([10, 20, 30, 40, 50, 60]) * doy
eofs.append(eof.EOFData(lat, long, eof1, eof2,
explained_variances=np.squeeze(explained_variances[:, doy - 1]),
eigenvalues=np.squeeze(eigenvalues[:, doy - 1]), no_observations=no_obs[doy - 1]))
target = eof.EOFDataForAllDOYs(eofs)
errors = []
if not np.all(target.explained_variance1_for_all_doys() == explained_variances[0, :]):
errors.append("Explained variance 1 incorrect")
if not np.all(target.explained_variance2_for_all_doys() == explained_variances[1, :]):
errors.append("Explained variance 2 incorrect")
if not np.all(target.eigenvalue1_for_all_doys() == eigenvalues[0, :]):
errors.append("Eigenvalue 1 incorrect")
if not np.all(target.eigenvalue2_for_all_doys() == eigenvalues[1, :]):
errors.append("Eigenvalue 2 incorrect")
if not np.all(target.total_explained_variance_for_all_doys() == np.sum(explained_variances, axis=0)):
errors.append("Total explained variance incorrect")
if not np.all(target.no_observations_for_all_doys() == no_obs):
errors.append("number of observations incorrect")
assert not errors, "errors occurred:\n{}".format("\n".join(errors))
def test_save_all_eofs_to_npzfile(tmp_path):
filename = tmp_path / "test.npz"
doys = tools.doy_list()
lat = np.array([-10., 0., 10.])
long = np.array([0., 5.])
explained_variances = np.array([np.arange(1, doys.size + 1, 1) + 111,
np.arange(1, doys.size + 1, 1) + 222,
np.arange(1, doys.size + 1, 1) + 333,
np.arange(1, doys.size + 1, 1) + 444,
np.arange(1, doys.size + 1, 1) + 555,
np.arange(1, doys.size + 1, 1) + 666])
eigenvalues = np.array([np.arange(1, doys.size + 1, 1) + 1111,
np.arange(1, doys.size + 1, 1) + 2222,
np.arange(1, doys.size + 1, 1) + 3333,
np.arange(1, doys.size + 1, 1) + 4444,
np.arange(1, doys.size + 1, 1) + 5555,
np.arange(1, doys.size + 1, 1) + 6666])
no_obs = doys * 5
eofs = []
for doy in doys:
eof1 = np.array([1, 2, 3, 4, 5, 6]) * doy
eof2 = np.array([10, 20, 30, 40, 50, 60]) * doy
eofs.append(eof.EOFData(lat, long, eof1, eof2, explained_variances=np.squeeze(explained_variances[:, doy - 1]),
eigenvalues=np.squeeze(eigenvalues[:, doy - 1]), no_observations=no_obs[doy - 1]))
target = eof.EOFDataForAllDOYs(eofs)
target.save_all_eofs_to_npzfile(filename)
errors = []
target_reloaded = eof.restore_all_eofs_from_npzfile(filename)
if not target_reloaded.eof_list == eofs:
errors.append("List of EOFData objects incorrect")
if not np.all(target_reloaded.lat == lat):
errors.append("Lat is incorrect")
if not np.all(target_reloaded.long == long):
errors.append("Long is incorrect")
if not target_reloaded.eofdata_for_doy(1) == eofs[0]:
errors.append("Sample EOF data is incorrect")
assert not errors, "errors occurred:\n{}".format("\n".join(errors))
def load_all_original_eofs_from_directory(dirname: Path) -> EOFDataForAllDOYs:
"""
Loads the EOF functions for all DOYs from the original file format.
The original EOFs are found here: ftp://ftp.cdc.noaa.gov/Datasets.other/MJO/eof1/ and
ftp://ftp.cdc.noaa.gov/Datasets.other/MJO/eof2/
Note that the EOFs are represented as pure vectors in the original treatment, so that a connection to the
individual locations on a world map is not obvious without any further knowledge. The corresponding grid is here
inserted hardcodedly.
:param dirname: Path to the directory, in which the EOFs for all DOYs are stored.
This path should contain the sub directories *eof1* and *eof2*, in which the 366 files each are located:
One file per day of the year.
:return: The original EOFs for all DOYs.
"""
eofs = []
for doy in doy_list():
eof = load_original_eofs_for_doy(dirname, doy)
eofs.append(eof)
return EOFDataForAllDOYs(eofs)
def calc_eofs_from_preprocessed_olr(
olrdata: olr.OLRData,
implementation: str = "internal",
strict_leap_year_treatment: bool = False) -> eof.EOFDataForAllDOYs:
"""
Calculates a series of EOF pairs: one pair for each DOY.
This is based on already preprocessed OLR. Note that it is recommended to use the function
:meth:`calc_eofs_from_olr` to cover the complete algorithm.
:param olrdata: the preprocessed OLR data, from which the EOFs are calculated.
:param implementation: Two options are available: First, "internal": uses the internal implementation of the EOF
approach. Second, "eofs_package": Uses the implementation of the external package :py:mod:`eofs`.
:param strict_leap_year_treatment: see description in :meth:`mjoindices.tools.find_doy_ranges_in_dates`.
:return: A pair of EOFs for each DOY. This series of EOFs has probably still to be postprocessed.
"""
if implementation == "eofs_package" and not eofs_package_available:
raise ValueError(
"Selected calculation with external eofs package, but package not available. Use "
"internal implementation or install eofs package")
doys = tools.doy_list()
eofs = []
for doy in doys:
print("Calculating EOFs for DOY %i" % doy)
if (implementation == "eofs_package"):
singleeof = calc_eofs_for_doy_using_eofs_package(
olrdata,
doy,
strict_leap_year_treatment=strict_leap_year_treatment)
else:
singleeof = calc_eofs_for_doy(
olrdata,
doy,
strict_leap_year_treatment=strict_leap_year_treatment)
eofs.append(singleeof)
return eof.EOFDataForAllDOYs(eofs)
def test_doy_list():
target = tools.doy_list()
assert np.all(target == np.arange(1, 367, 1))
def test_calc_comparison_stats_for_eofs_all_doys():
doys = tools.doy_list()
lat = np.array([-10., 0., 10.])
long = np.array([0., 5.])
errors = []
eofs_reference = []
eofs_data = []
for doy in doys:
eof1 = np.array([1, 2, 3, 4, 5, 6]) * doy
eof2 = np.array([10, 20, 30, 40, 50, 60]) * doy
eofs_reference.append(eof.EOFData(lat, long, eof1, eof2))
if doy == 3:
eof1 = -1 * eof1
if doy == 4:
eof2 = -1 * eof2
eofs_data.append(eof.EOFData(lat, long, eof1, eof2))
reference = eof.EOFDataForAllDOYs(eofs_reference)
data = eof.EOFDataForAllDOYs(eofs_data)
corr, diff_mean, diff_std, diff_abs_percent68, diff_abs_percent95, diff_abs_percent99 = \
evalt.calc_comparison_stats_for_eofs_all_doys(reference, data, 1, exclude_doy366=False, percentage=False,
do_print=False)
if not (np.allclose(corr[:2], 1) and np.allclose(corr[3:], 1)):
errors.append("EOF1: Correlations wrong")
if not np.isclose(corr[2], -1.):
errors.append("EOF1: Correlation for DOY 3 wrong")
if not (np.allclose(diff_mean[:2], 0) and np.allclose(diff_mean[3:], 0)):
errors.append("EOF1: Mean wrong")
if not diff_mean[2] < 0:
errors.append("EOF1: Mean for DOY 3 wrong")
if not (np.allclose(diff_std[:2], 0) and np.allclose(diff_std[3:], 0)):
errors.append("EOF1: StdDev wrong")
if not diff_std[2] > 0:
errors.append("EOF1: StdDev for DOY 3 wrong")
if not (np.allclose(diff_abs_percent68[:2], 0)
and np.allclose(diff_abs_percent68[3:], 0)):
errors.append("EOF1: 68% Percentile wrong")
if not diff_abs_percent68[2] > 0:
errors.append("EOF1: 68% Percentile for DOY 3 wrong")
if not (np.allclose(diff_abs_percent95[:2], 0)
and np.allclose(diff_abs_percent95[3:], 0)):
errors.append("EOF1: 95% Percentile wrong")
if not diff_abs_percent95[2] > 0:
errors.append("EOF1: 95% Percentile for DOY 3 wrong")
if not (np.allclose(diff_abs_percent99[:2], 0)
and np.allclose(diff_abs_percent99[3:], 0)):
errors.append("EOF1: 99% Percentile wrong")
if not diff_abs_percent99[2] > 0:
errors.append("EOF1: 99% Percentile for DOY 3 wrong")
corr, diff_mean, diff_std, diff_abs_percent68, diff_abs_percent95, diff_abs_percent99 = evalt.calc_comparison_stats_for_eofs_all_doys(
reference,
data,
2,
exclude_doy366=False,
percentage=False,
do_print=False)
if not (np.allclose(corr[:3], 1) and np.allclose(corr[4:], 1)):
errors.append("EOF2: Correlations wrong")
if not np.isclose(corr[3], -1.):
errors.append("EOF2: Correlation for DOY 4 wrong")
if not (np.allclose(diff_mean[:3], 0) and np.allclose(diff_mean[4:], 0)):
errors.append("EOF2: Mean wrong")
if not diff_mean[3] < 0:
errors.append("EOF2: Mean for DOY 4 wrong")
if not (np.allclose(diff_std[:3], 0) and np.allclose(diff_std[4:], 0)):
errors.append("EOF2: StdDev wrong")
if not diff_std[3] > 0:
errors.append("EOF2: StdDev for DOY 4 wrong")
if not (np.allclose(diff_abs_percent68[:3], 0)
and np.allclose(diff_abs_percent68[4:], 0)):
errors.append("EOF2: 68% Percentile wrong")
if not diff_abs_percent68[3] > 0:
errors.append("EOF2: 68% Percentile for DOY 4 wrong")
if not (np.allclose(diff_abs_percent95[:3], 0)
and np.allclose(diff_abs_percent95[4:], 0)):
errors.append("EOF2: 95% Percentile wrong")
if not diff_abs_percent95[3] > 0:
errors.append("EOF2: 95% Percentile for DOY 4 wrong")
if not (np.allclose(diff_abs_percent99[:3], 0)
and np.allclose(diff_abs_percent99[4:], 0)):
errors.append("EOF2: 99% Percentile wrong")
if not diff_abs_percent99[3] > 0:
errors.append("EOF2: 99% Percentile for DOY 4 wrong")
assert not errors, "errors occurred:\n{}".format("\n".join(errors))
def plot_comparison_stats_for_eofs_all_doys(recalc_eof: eof.EOFDataForAllDOYs,
orig_eof: eof.EOFDataForAllDOYs,
exclude_doy366: bool = False,
do_print: bool = False) -> Figure:
"""
Plots extended comparison statistics between calculated EOFs and references EOFs.
Statistics of the differences will be shown for each DOY (DOY on the abscissa) and with one line each for EOF1 and
EOF2.
:param recalc_eof: The EOFs to compare.
:param orig_eof: The reference eofs.
:param exclude_doy366: If True, DOY 366 will be included in the plot (sometimes worse agreement depending on the
leap year treatment mode).
:param do_print: If True, some characteristic values will we printed to the console.
:return: The figure handle
"""
doys = tools.doy_list()
if exclude_doy366:
doys = doys[:-1]
xlim = (0, doys[-1])
(corr_1, diff_mean_1, diff_std_1, diff_abs_percent68_1, diff_abs_percent95_1, diff_abs_percent99_1)\
= calc_comparison_stats_for_eofs_all_doys(orig_eof, recalc_eof, exclude_doy366=exclude_doy366, eof_number=1,
percentage=False, do_print=do_print)
(corr_2, diff_mean_2, diff_std_2, diff_abs_percent68_2, diff_abs_percent95_2, diff_abs_percent99_2)\
= calc_comparison_stats_for_eofs_all_doys(orig_eof, recalc_eof, exclude_doy366=exclude_doy366, eof_number=2,
percentage=False, do_print=do_print)
fig_id = "plot_comparison_stats_for_eofs_all_doys"
fig, axs = plt.subplots(4,
1,
num=fig_id,
clear=True,
figsize=(9, 9),
dpi=150)
plt.subplots_adjust(hspace=0.5)
fig.suptitle("Comparison of EOFs for all DOYs")
ax = axs[0]
ax.set_title("Correlation")
p11, = ax.plot(doys, corr_1, label="EOF1", color="blue")
p12, = ax.plot(doys, corr_2, label="EOF2", color="green")
ax.set_xlim(xlim)
ax.set_ylabel("Correlation")
fig.legend(handles=[p11, p12])
ax = axs[1]
ax.set_title("Mean of differences of EOF vector elements")
p21, = ax.plot(doys, diff_mean_1, label="EOF1", color="blue")
p22, = ax.plot(doys, diff_mean_2, label="EOF2", color="green")
ax.set_xlim(xlim)
ax.set_ylabel(r"Mean [$\mathrm{W/m^2}$]")
ax = axs[2]
ax.set_title("Standard deviation of differences of EOF vector elements")
p31, = ax.plot(doys, diff_std_1, label="EOF1", color="blue")
p32, = ax.plot(doys, diff_std_2, label="EOF2", color="green")
ax.set_xlim(xlim)
ax.set_ylabel(r"Std.Dev. [$\mathrm{W/m^2}$]")
ax = axs[3]
ax.set_title("Percentiles of absolute differences of EOF vector elements")
p31, = ax.plot(doys, diff_abs_percent99_1, label="99% EOF1", color="blue")
p32, = ax.plot(doys, diff_abs_percent99_2, label="99% EOF2", color="green")
p33, = ax.plot(doys,
diff_abs_percent95_1,
label="EOF1",
color="blue",
linestyle="--")
p34, = ax.plot(doys,
diff_abs_percent95_2,
label="EOF2",
color="green",
linestyle="--")
p35, = ax.plot(doys,
diff_abs_percent68_1,
label="EOF1",
color="blue",
linestyle=":")
p36, = ax.plot(doys,
diff_abs_percent68_2,
label="EOF2",
color="green",
linestyle=":")
ax.set_xlim(xlim)
ax.set_ylabel(r"Percentiles [$\mathrm{W/m^2}$]")
ax.legend(labels=["99%", "95%", "68%"],
handles=[p31, p33, p35],
loc="upper right")
ax.set_xlabel("Day of year")
return fig
def calc_comparison_stats_for_eofs_all_doys(
eofs_ref: eof.EOFDataForAllDOYs,
eofs: eof.EOFDataForAllDOYs,
eof_number,
exclude_doy366: bool = False,
percentage: bool = False,
do_print: bool = False) -> typing.Tuple:
"""
Calculates extended comparison statistics between calculated EOFs and reference EOFs.
:param recalc_eof: The EOFs to validate.
:param orig_eof: The reference EOFs.
:param exclude_doy366: If True, DOY 366 will be included in the calculation (sometimes worse agreement depending on
the leap year treatment mode).
:param percentage: If True: calculation results will be in percent of the mean of the absolute reference values.
:param do_print: If True, some characteristic values will we printed to the console.
:return: A tuple containing arrays with 366 or 365 elements each for the following quantities: correlation, mean
of the differences, standard deviation of the differences, and percentiles of the absolute differences
for 68%, 95%, and 99%.
"""
if eof_number != 1 and eof_number != 2:
raise ValueError("Argument eof_number must be 1 or 2.")
doys = tools.doy_list()
if exclude_doy366:
doys = doys[:-1]
corr = np.empty(doys.size)
diff_mean = np.empty(doys.size)
diff_std = np.empty(doys.size)
diff_abs_percent68 = np.empty(doys.size)
diff_abs_percent95 = np.empty(doys.size)
diff_abs_percent99 = np.empty(doys.size)
for idx, doy in enumerate(doys):
if eof_number == 1:
eof_ref = eofs_ref.eof1vector_for_doy(doy)
eof_test = eofs.eof1vector_for_doy(doy)
elif eof_number == 2:
eof_ref = eofs_ref.eof2vector_for_doy(doy)
eof_test = eofs.eof2vector_for_doy(doy)
(corr_single, diff_mean_single, diff_std_single, diff_vec_single, diff_abs_percent68_single,
diff_abs_percent95_single, diff_abs_percent99_single) \
= calc_vector_agreement(eof_ref, eof_test, percentage=percentage, do_print=False)
corr[idx] = corr_single
diff_mean[idx] = diff_mean_single
diff_std[idx] = diff_std_single
diff_abs_percent68[idx] = diff_abs_percent68_single
diff_abs_percent95[idx] = diff_abs_percent95_single
diff_abs_percent99[idx] = diff_abs_percent99_single
if do_print:
print("########## Summary of EOF comparison for all DOYs (EOF %i)" %
eof_number)
print("Worst Correlation (at DOY %i): %1.4f" %
(doys[np.argmin(corr)], np.amin(corr)))
print(
"Worst 99%% percentile (at DOY %i): %1.4f" %
(doys[np.argmax(diff_abs_percent99)], np.amax(diff_abs_percent99)))
print(
"Worst 68%% percentile (at DOY %i): %1.4f" %
(doys[np.argmax(diff_abs_percent68)], np.amax(diff_abs_percent68)))
return corr, diff_mean, diff_std, diff_abs_percent68, diff_abs_percent95, diff_abs_percent99
