def transformed(self, X, wavenumbers):
# wavenumber have to be input as sorted
# about 85% of time in __call__ function is spent is lstsq
# compute average spectrum from the reference
ref_X = np.atleast_2d(spectra_mean(self.reference.X))
def interpolate_to_data(other_xs, other_data):
# all input data needs to be interpolated (and NaNs removed)
interpolated = interp1d_with_unknowns_numpy(
other_xs, other_data, wavenumbers)
# we know that X is not NaN. same handling of reference as of X
interpolated, _ = nan_extend_edges_and_interpolate(
wavenumbers, interpolated)
return interpolated
ref_X = interpolate_to_data(getx(self.reference), ref_X)
wei_X = weighted_wavenumbers(self.weights, wavenumbers)
N = wavenumbers.shape[0]
m0 = -2.0 / (wavenumbers[0] - wavenumbers[N - 1])
c_coeff = 0.5 * (wavenumbers[0] + wavenumbers[N - 1])
n_badspec = len(self.badspectra) if self.badspectra is not None else 0
if self.badspectra:
badspectra_X = interpolate_to_data(getx(self.badspectra),
self.badspectra.X)
M = []
for x in range(0, self.order + 1):
M.append((m0 * (wavenumbers - c_coeff))**x)
for y in range(0, n_badspec):
M.append(badspectra_X[y])
M.append(ref_X) # always add reference spectrum to the model
n_add_model = len(M)
M = np.vstack(
M
).T # M is for the correction, for par. estimation M_weighted is used
M_weighted = M * wei_X.T
newspectra = np.zeros((X.shape[0], X.shape[1] + n_add_model))
for i, rawspectrum in enumerate(X):
rawspectrumW = (rawspectrum * wei_X)[0]
m = np.linalg.lstsq(M_weighted, rawspectrum, rcond=-1)[0]
corrected = rawspectrum
for x in range(0, self.order + 1 + n_badspec):
corrected = (corrected - (m[x] * M[:, x]))
if self.scaling:
corrected = corrected / m[self.order + 1 + n_badspec]
corrected[np.isinf(
corrected)] = np.nan # fix values caused by zero weights
corrected = np.hstack(
(corrected, m)) # append the model parameters
newspectra[i] = corrected
return newspectra
def transformed(self, X, wavenumbers):
# wavenumber have to be input as sorted
# about 85% of time in __call__ function is spent is lstsq
# compute average spectrum from the reference
ref_X = np.atleast_2d(spectra_mean(self.reference.X))
def interpolate_to_data(other_xs, other_data):
# all input data needs to be interpolated (and NaNs removed)
interpolated = interp1d_with_unknowns_numpy(other_xs, other_data, wavenumbers)
# we know that X is not NaN. same handling of reference as of X
interpolated, _ = nan_extend_edges_and_interpolate(wavenumbers, interpolated)
return interpolated
ref_X = interpolate_to_data(getx(self.reference), ref_X)
if self.weights:
# interpolate reference to the data
wei_X = interp1d_with_unknowns_numpy(getx(self.weights), self.weights.X, wavenumbers)
# set whichever weights are undefined (usually at edges) to zero
wei_X[np.isnan(wei_X)] = 0
else:
wei_X = np.ones((1, len(wavenumbers)))
N = wavenumbers.shape[0]
m0 = - 2.0 / (wavenumbers[0] - wavenumbers[N - 1])
c_coeff = 0.5 * (wavenumbers[0] + wavenumbers[N - 1])
n_badspec = len(self.badspectra) if self.badspectra is not None else 0
if self.badspectra:
badspectra_X = interpolate_to_data(getx(self.badspectra), self.badspectra.X)
M = []
for x in range(0, self.order+1):
M.append((m0 * (wavenumbers - c_coeff)) ** x)
for y in range(0, n_badspec):
M.append(badspectra_X[y])
M.append(ref_X) # always add reference spectrum to the model
n_add_model = len(M)
M = np.vstack(M).T # M is for the correction, for par. estimation M_weighted is used
M_weighted = M*wei_X.T
newspectra = np.zeros((X.shape[0], X.shape[1] + n_add_model))
for i, rawspectrum in enumerate(X):
rawspectrumW = (rawspectrum*wei_X)[0]
m = np.linalg.lstsq(M_weighted, rawspectrum, rcond=-1)[0]
corrected = rawspectrum
for x in range(0, self.order+1+n_badspec):
corrected = (corrected - (m[x] * M[:, x]))
if self.scaling:
corrected = corrected/m[self.order+1+n_badspec]
corrected[np.isinf(corrected)] = np.nan # fix values caused by zero weights
corrected = np.hstack((corrected, m)) # append the model parameters
newspectra[i] = corrected
return newspectra
def transformed(self, X, wavenumbers):
# about 85% of time in __call__ function is spent is lstsq
# compute average spectrum from the reference
ref_X = np.atleast_2d(spectra_mean(self.reference.X))
# interpolate reference to the data
ref_X = interp1d_with_unknowns_numpy(getx(self.reference), ref_X,
wavenumbers)
# we know that X is not NaN. same handling of reference as of X
ref_X, _ = nan_extend_edges_and_interpolate(wavenumbers, ref_X)
if self.weights:
# interpolate reference to the data
wei_X = interp1d_with_unknowns_numpy(getx(self.weights),
self.weights.X, wavenumbers)
# set whichever weights are undefined (usually at edges) to zero
wei_X[np.isnan(wei_X)] = 0
else:
wei_X = np.ones((1, len(wavenumbers)))
N = wavenumbers.shape[0]
m0 = -2.0 / (wavenumbers[0] - wavenumbers[N - 1])
c_coeff = 0.5 * (wavenumbers[0] + wavenumbers[N - 1])
M = []
for x in range(0, self.order + 1):
M.append((m0 * (wavenumbers - c_coeff))**x)
M.append(ref_X) # always add reference spectrum to the model
n_add_model = len(M)
M = np.vstack(
M
).T # M is needed below for the correction, for par estimation M_weigheted is used
M_weighted = M * wei_X.T
newspectra = np.zeros((X.shape[0], X.shape[1] + n_add_model))
for i, rawspectrum in enumerate(X):
rawspectrumW = (rawspectrum * wei_X)[0]
m = np.linalg.lstsq(M_weighted, rawspectrum)[0]
corrected = rawspectrum
for x in range(0, self.order + 1):
corrected = (corrected - (m[x] * M[:, x]))
if self.scaling:
corrected = corrected / m[self.order + 1]
corrected[np.isinf(
corrected
)] = np.nan # fix values which can be caused by zero weights
corrected = np.hstack(
(corrected, m)) # append the model parameters
newspectra[i] = corrected
return newspectra
def update_reference_info(self):
if not self.reference:
self.reference_curve.hide()
self.reference_info.setText("Reference: missing!")
self.reference_info.setStyleSheet("color: red")
else:
rinfo = "mean of %d spectra" % len(self.reference) \
if len(self.reference) > 1 else "1 spectrum"
self.reference_info.setText("Reference: " + rinfo)
self.reference_info.setStyleSheet("color: black")
X_ref = spectra_mean(self.reference.X)
x = getx(self.reference)
xsind = np.argsort(x)
self.reference_curve.setData(x=x[xsind], y=X_ref[xsind])
self.reference_curve.setVisible(self.scaling)
def __init__(self,
reference=None,
weights=None,
ncomp=False,
n0=np.linspace(1.1, 1.4, 10),
a=np.linspace(2, 7.1, 10),
h=0.25,
max_iter=30,
fixed_iter=False,
positive_reference=True,
output_model=False,
ranges=None):
# the first non-kwarg can not be a data table (Preprocess limitations)
# ranges could be a list like this [[800, 1000], [1300, 1500]]
if reference is None:
raise MissingReferenceException()
self.reference = reference
self.weights = weights
self.ncomp = ncomp
self.output_model = output_model
explainedVariance = 99.96
self.maxNiter = max_iter
self.fixedNiter = fixed_iter
self.positiveRef = positive_reference
self.n0 = n0
self.a = a
self.h = h
self.alpha0 = (4 * np.pi * self.a * (self.n0 - 1)) * 1e-6
self.gamma = self.h * np.log(10) / (4 * np.pi * 0.5 * np.pi *
(self.n0 - 1) * self.a * 1e-6)
if not self.ncomp:
ref_X = np.atleast_2d(spectra_mean(self.reference.X))
wavenumbers_ref = np.array(sorted(getx(self.reference)))
ref_X = interpolate_to_data(getx(self.reference), ref_X,
wavenumbers_ref)
ref_X = ref_X[0]
self.ncomp = cal_ncomp(ref_X, wavenumbers_ref, explainedVariance,
self.alpha0, self.gamma)
else:
self.explainedVariance = False
def transformed(self, X, wavenumbers):
# wavenumber have to be input as sorted
# compute average spectrum from the reference
def make_basic_emsc_mod(ref_X):
N = wavenumbers.shape[0]
m0 = -2.0 / (wavenumbers[0] - wavenumbers[N - 1])
c_coeff = 0.5 * (wavenumbers[0] + wavenumbers[N - 1])
M_basic = []
for x in range(0, 3):
M_basic.append((m0 * (wavenumbers - c_coeff))**x)
M_basic.append(ref_X) # always add reference spectrum to the model
M_basic = np.vstack(M_basic).T
return M_basic
def cal_emsc_basic(M_basic, spectrum):
m = np.linalg.lstsq(M_basic, spectrum, rcond=-1)[0]
corrected = spectrum
for x in range(0, 3):
corrected = (corrected - (m[x] * M_basic[:, x]))
corrected = corrected / m[3]
scaled_spectrum = corrected
return scaled_spectrum
def make_emsc_model(badspectra, referenceSpec):
M = np.ones([len(wavenumbers), self.ncomp + 2])
M[:, 1:self.ncomp + 1] = np.array(
[spectrum for spectrum in badspectra.T])
M[:, self.ncomp + 1] = referenceSpec
return M
def cal_emsc(M, X):
correctedspectra = np.zeros((X.shape[0], X.shape[1] + M.shape[1]))
for i, rawspectrum in enumerate(X):
m = np.linalg.lstsq(M, rawspectrum, rcond=-1)[0]
corrected = rawspectrum
for x in range(0, 1 + self.ncomp):
corrected = (corrected - (m[x] * M[:, x]))
corrected = corrected / m[1 + self.ncomp]
corrected[np.isinf(
corrected)] = np.nan # fix values caused by zero weights
corrected = np.hstack(
(corrected, m)) # append the model parameters
correctedspectra[i] = corrected
params = correctedspectra[:, -(self.ncomp + 2):]
res = X - np.dot(params,
M.T) # Have to check if this is correct FIXME
return correctedspectra, res
def iteration_step(spectrum, reference, wavenumbers, M_basic, alpha0,
gamma):
# scale with basic EMSC:
reference = cal_emsc_basic(M_basic, reference)
# some BLAS implementation can raise an exception in the upper call (MKL)
# while some other only return an array of NaN (OpenBLAS), therefore
# raise an exception manually
if np.all(np.isnan(reference)):
raise np.linalg.LinAlgError()
# Apply weights
reference = reference * wei_X
reference = reference[0]
# set negative parts to zero
nonzeroReference = reference.copy()
nonzeroReference[nonzeroReference < 0] = 0
if self.positiveRef:
reference = nonzeroReference
# calculate Qext-curves
nprs, nkks = calculate_complex_n(nonzeroReference, wavenumbers)
Qext = calculate_Qext_curves(nprs, nkks, alpha0, gamma,
wavenumbers)
Qext = orthogonalize_Qext(Qext, reference)
badspectra = compress_Mie_curves(Qext, self.ncomp)
# build ME-EMSC model
M = make_emsc_model(badspectra, reference)
# calculate parameters and corrected spectra
newspectrum, res = cal_emsc(M, spectrum)
return newspectrum, res
def iterate(spectra, correctedFirsIteration, residualsFirstIteration,
wavenumbers, M_basic, alpha0, gamma):
newspectra = np.full(correctedFirsIteration.shape, np.nan)
numberOfIterations = np.full(spectra.shape[0], np.nan)
residuals = np.full(spectra.shape, np.nan)
RMSEall = np.full([spectra.shape[0]], np.nan)
for i in range(correctedFirsIteration.shape[0]):
corrSpec = correctedFirsIteration[i]
rawSpec = spectra[i, :]
rawSpec = rawSpec.reshape(1, -1)
RMSE = [
round(
np.sqrt((1 / len(residualsFirstIteration[i, :])) *
np.sum(residualsFirstIteration[i, :]**2)), 4)
]
for iterationNumber in range(2, self.maxNiter + 1):
try:
newSpec, res = iteration_step(
rawSpec, corrSpec[:-self.ncomp - 2], wavenumbers,
M_basic, alpha0, gamma)
except np.linalg.LinAlgError:
newspectra[i, :] = np.full(
[rawSpec.shape[1] + self.ncomp + 2], np.nan)
residuals[i, :] = np.full(rawSpec.shape, np.nan)
RMSEall[i] = np.nan
break
corrSpec = newSpec[0, :]
rmse = round(
np.sqrt((1 / len(res[0, :])) * np.sum(res**2)), 4)
RMSE.append(rmse)
# Stop criterion
if iterationNumber == self.maxNiter:
newspectra[i, :] = corrSpec
numberOfIterations[i] = iterationNumber
residuals[i, :] = res
RMSEall[i] = RMSE[-1]
break
elif self.fixedNiter and iterationNumber < self.fixedNiter:
continue
elif iterationNumber == self.maxNiter or iterationNumber == self.fixedNiter:
newspectra[i, :] = corrSpec
numberOfIterations[i] = iterationNumber
residuals[i, :] = res
RMSEall[i] = RMSE[-1]
break
elif iterationNumber > 2 and self.fixedNiter == False:
if (rmse == RMSE[-2]
and rmse == RMSE[-3]) or rmse > RMSE[-2]:
newspectra[i, :] = corrSpec
numberOfIterations[i] = iterationNumber
residuals[i, :] = res
RMSEall[i] = RMSE[-1]
break
return newspectra, RMSEall, numberOfIterations
ref_X = np.atleast_2d(spectra_mean(self.reference.X))
ref_X = interpolate_to_data(getx(self.reference), ref_X, wavenumbers)
ref_X = ref_X[0]
wei_X = weighted_wavenumbers(self.weights, wavenumbers)
ref_X = ref_X * wei_X
ref_X = ref_X[0]
nonzeroReference = ref_X
nonzeroReference[nonzeroReference < 0] = 0
if self.positiveRef:
ref_X = nonzeroReference
resonant = True # Possibility for using the 2008 version
if resonant: # if this should be any point, we need to terminate after 1 iteartion for the non-resonant one
nprs, nkks = calculate_complex_n(ref_X, wavenumbers)
else:
npr = np.zeros(len(wavenumbers))
nprs = npr / (wavenumbers * 100)
nkks = np.zeros(len(wavenumbers))
# For the first iteration, make basic EMSC model
M_basic = make_basic_emsc_mod(
ref_X
) # Consider to make the M_basic in the init since this one does not change.
# Calculate scattering curves for ME-EMSC
Qext = calculate_Qext_curves(nprs, nkks, self.alpha0, self.gamma,
wavenumbers)
Qext = orthogonalize_Qext(Qext, ref_X)
badspectra = compress_Mie_curves(Qext, self.ncomp)
# Establish ME-EMSC model
M = make_emsc_model(badspectra, ref_X)
# Correcting all spectra at once for the first iteration
newspectra, res = cal_emsc(M, X)
if self.fixedNiter == 1 or self.maxNiter == 1:
res = np.array(res)
numberOfIterations = np.ones([1, newspectra.shape[0]])
RMSEall = [
round(np.sqrt((1 / res.shape[1]) * np.sum(res[specNum, :]**2)),
4) for specNum in range(newspectra.shape[0])
] # ADD RESIDUALS!!!!! FIXME
newspectra = np.hstack(
(newspectra, numberOfIterations.reshape(-1, 1),
np.array(RMSEall).reshape(-1, 1)))
return newspectra
# Iterate
newspectra, RMSEall, numberOfIterations = iterate(
X, newspectra, res, wavenumbers, M_basic, self.alpha0, self.gamma)
newspectra = np.hstack((newspectra, numberOfIterations.reshape(-1, 1),
RMSEall.reshape(-1, 1)))
return newspectra