def test_sklearn_mcr_errors():
# Providing C in fit_kwargs and S^T to fit without both C_fix and St_fix
with pytest.raises(TypeError):
mcrar = McrAR(fit_kwargs={'C': np.random.randn(10, 3), 'c_fix': [0]})
# Only c_fix
mcrar.fit(np.random.randn(10, 5), ST=np.random.randn(3, 5))
def test_props_features_samples_targets(dataset):
""" Test mcrar properties for features, targets, samples """
C_known, D_known, St_known = dataset
mcrar = McrAR()
mcrar.fit(D_known, ST=St_known)
assert mcrar.n_targets == C_known.shape[-1] # n_components
assert mcrar.n_samples == D_known.shape[0]
assert mcrar.n_features == D_known.shape[-1]
def test_mcr_tol_err_change(dataset):
""" Test MCR exits due error increasing by a value """
C_known, D_known, St_known = dataset
mcrar = McrAR(max_iter=50,
c_regr='OLS',
st_regr='OLS',
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_increase=None,
tol_n_increase=None,
tol_err_change=1e-20,
tol_n_above_min=None)
mcrar.fit(D_known, C=C_known)
assert mcrar.exit_tol_err_change
def test_sklearn_mcr_c_semilearned():
""" Test when C items are fixed, i.e., enforced to be the same as the input, always """
M = 21
N = 21
P = 101
n_components = 3
C_img = np.zeros((M, N, n_components))
C_img[..., 0] = np.dot(np.ones((M, 1)), np.linspace(0, 1, N)[None, :])
C_img[..., 1] = np.dot(np.linspace(0, 1, M)[:, None], np.ones((1, N)))
C_img[..., 2] = 1 - C_img[..., 0] - C_img[..., 1]
C_img = C_img / C_img.sum(axis=-1)[:, :, None]
St_known = np.zeros((n_components, P))
St_known[0, 30:50] = 1
St_known[1, 50:70] = 2
St_known[2, 70:90] = 3
St_known += 1
C_known = C_img.reshape((-1, n_components))
D_known = np.dot(C_known, St_known)
C_guess = 1 * C_known
C_guess[:, 2] = np.abs(np.random.randn(int(M * N)) + 0.1)
mcrar = McrAR(max_iter=50,
tol_increase=100,
tol_n_increase=10,
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_err_change=1e-10,
fit_kwargs={
'C': C_guess,
'c_fix': [0, 1]
})
mcrar.fit(D_known)
assert_equal(mcrar.C_[:, 0], C_known[:, 0])
assert_equal(mcrar.C_[:, 1], C_known[:, 1])
def test_mcr_max_iterations(dataset):
""" Test MCR exits at max_iter"""
C_known, D_known, St_known = dataset
# Seeding with a constant of 0.1 for C, actually leads to a bad local
# minimum; thus, the err_change gets really small with a relatively bad
# error. The tol_err_change is set to None, so it makes it to max_iter.
mcrar = McrAR(max_iter=50,
c_regr='OLS',
st_regr='OLS',
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_increase=None,
tol_n_increase=None,
tol_err_change=None,
tol_n_above_min=None)
mcrar.fit(D_known, C=C_known * 0 + 0.1)
assert mcrar.exit_max_iter_reached
def test_mcr_tol_increase(dataset):
""" Test MCR exits due error increasing above a tolerance fraction"""
C_known, D_known, St_known = dataset
# Seeding with a constant of 0.1 for C, actually leads to a bad local
# minimum; thus, the err_change gets really small with a relatively bad
# error.
mcrar = McrAR(max_iter=50,
c_regr='OLS',
st_regr='OLS',
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_increase=0,
tol_n_increase=None,
tol_err_change=None,
tol_n_above_min=None)
mcrar.fit(D_known, C=C_known * 0 + 0.1)
assert mcrar.exit_tol_increase
def test_mcr_tol_n_above_min(dataset):
"""
Test MCR exits due to half-terating n times with error above the minimum error.
Note: On some CI systems, the minimum err bottoms out; thus, tol_n_above_min
needed to be set to 0 to trigger a break.
"""
C_known, D_known, St_known = dataset
mcrar = McrAR(max_iter=50,
c_regr='OLS',
st_regr='OLS',
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_increase=None,
tol_n_increase=None,
tol_err_change=None,
tol_n_above_min=0)
mcrar.fit(D_known, C=C_known * 0 + 0.1)
assert mcrar.exit_tol_n_above_min
def test_sklearn_mcr_semilearned_both_c_st():
"""
Test the special case when C & ST are provided, requiring C-fix ST-fix to
be provided
"""
M = 21
N = 21
P = 101
n_components = 3
C_img = np.zeros((M, N, n_components))
C_img[..., 0] = np.dot(np.ones((M, 1)), np.linspace(0.1, 1, N)[None, :])
C_img[..., 1] = np.dot(np.linspace(0.1, 1, M)[:, None], np.ones((1, N)))
C_img[..., 2] = 1 - C_img[..., 0] - C_img[..., 1]
C_img = C_img / C_img.sum(axis=-1)[:, :, None]
St_known = np.zeros((n_components, P))
St_known[0, 30:50] = 1
St_known[1, 50:70] = 2
St_known[2, 70:90] = 3
St_known += 1
C_known = C_img.reshape((-1, n_components))
D_known = np.dot(C_known, St_known)
C_guess = 1 * C_known
C_guess[:, 2] = np.abs(np.random.randn(int(M * N)))
mcrar = McrAR(max_iter=50,
tol_increase=100,
tol_n_increase=10,
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_err_change=1e-10,
fit_kwargs={
'C': C_guess,
'ST': St_known,
'c_fix': [0, 1],
'st_fix': [0]
})
mcrar.fit(D_known, c_first=True)
assert_equal(mcrar.C_[:, 0], C_known[:, 0])
assert_equal(mcrar.C_[:, 1], C_known[:, 1])
assert_equal(mcrar.ST_[0, :], St_known[0, :])
# ST-solve first
mcrar.fit(D_known,
C=C_guess,
ST=St_known,
c_fix=[0, 1],
st_fix=[0],
c_first=False)
assert_equal(mcrar.C_[:, 0], C_known[:, 0])
assert_equal(mcrar.C_[:, 1], C_known[:, 1])
assert_equal(mcrar.ST_[0, :], St_known[0, :])
def test_mcr_st_semilearned():
""" Test when St items are fixed, i.e., enforced to be the same as the input, always """
M = 21
N = 21
P = 101
n_components = 3
C_img = np.zeros((M, N, n_components))
C_img[..., 0] = np.dot(np.ones((M, 1)), np.linspace(0, 1, N)[None, :])
C_img[..., 1] = np.dot(np.linspace(0, 1, M)[:, None], np.ones((1, N)))
C_img[..., 2] = 1 - C_img[..., 0] - C_img[..., 1]
C_img = C_img / C_img.sum(axis=-1)[:, :, None]
St_known = np.zeros((n_components, P))
St_known[0, 30:50] = 1
St_known[1, 50:70] = 2
St_known[2, 70:90] = 3
St_known += 1
C_known = C_img.reshape((-1, n_components))
D_known = np.dot(C_known, St_known)
ST_guess = 1 * St_known
ST_guess[2, :] = np.random.randn(P)
mcrar = McrAR(max_iter=50,
tol_increase=100,
tol_n_increase=10,
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_err_change=1e-10)
mcrar.fit(D_known, ST=ST_guess, st_fix=[0, 1])
assert_equal(mcrar.ST_[0, :], St_known[0, :])
assert_equal(mcrar.ST_[1, :], St_known[1, :])
def test_sklearn_mcr_ideal_default(dataset):
""" Provides C/St_known so optimal should be 1 iteration """
C_known, D_known, St_known = dataset
mcrar = McrAR(fit_kwargs={'ST': St_known})
mcrar.fit(D_known)
assert_equal(1, mcrar.n_iter_opt)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
mcrar = McrAR(fit_kwargs={'C': C_known})
mcrar.fit(D_known)
assert_equal(2, mcrar.n_iter_opt)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
def test_sklearn_mcr_tol_n_increase(dataset):
"""
Test MCR exits due iterating n times with an increase in error
Note: On some CI systems, the minimum err bottoms out; thus, tol_n_above_min
needed to be set to 0 to trigger a break.
"""
C_known, D_known, St_known = dataset
mcrar = McrAR(max_iter=50,
c_regr='OLS',
st_regr='OLS',
st_constraints=[ConstraintNonneg()],
c_constraints=[ConstraintNonneg(),
ConstraintNorm()],
tol_increase=None,
tol_n_increase=0,
tol_err_change=None,
tol_n_above_min=None,
fit_kwargs={'C': C_known * 0 + 0.1})
mcrar.fit(D_known)
assert mcrar.exit_tol_n_increase
def test_mcr_ideal_default(dataset):
""" Provides C/St_known so optimal should be 1 iteration """
C_known, D_known, St_known = dataset
mcrar = McrAR()
mcrar.fit(D_known, ST=St_known)
assert_equal(1, mcrar.n_iter_opt)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
mcrar.fit(D_known, C=C_known)
assert_equal(2, mcrar.n_iter_opt)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
#ica=NMF(n_components=2)
DN = np.copy(D)
DN[DN < 1e-100] = 1e-100
C_est = ica.fit_transform(DN)
#flip negative peak
C_est[:, np.max(C_est, axis=0) < -1 * np.min(C_est, axis=0)] *= -1
#remove base line
C_est -= np.min(C_est, axis=0)
plt.figure()
plt.title("Initial estimate by FastICA")
plt.plot(C_est)
plt.pause(0.1)
from pymcr.mcr import McrAR
mcrar = McrAR()
mcrar.fit(D, C=C_est, verbose=True)
plt.figure()
plt.title("result by MCR")
plt.plot(mcrar.C_opt_ * np.sum(mcrar.ST_opt_, axis=1))
plt.plot(gA, label="trueA")
plt.plot(gB, label="trueB")
plt.legend()
plt.pause(0.1)
if True:
#BEMG関数制約にregressorを使う
#処理がとっても重いが、収束はする しかし、最適解にいくとは限らない
c_regr = RegArray([FunBEMG(), FunBEMG()])
mcrar_c = McrAR(c_regr=c_regr,
st_constraints.append(ConstraintNonneg())
if 'ConstraintCumsumNonneg' in st_l:
st_constraints.append(ConstraintCumsumNonneg())
if 'ConstraintNorm' in st_l:
st_constraints.append(ConstraintNorm())
# print(c_constraints,st_constraints)
logger = logging.getLogger('pymcr')
logger.setLevel(logging.DEBUG)
stdout_handler = logging.StreamHandler(stream=sys.stdout)
stdout_format = logging.Formatter('%(message)s')
stdout_handler.setFormatter(stdout_format)
logger.addHandler(stdout_handler)
mcrar = McrAR(
max_iter=max_iter,
st_regr=NNLS(),
c_regr=OLS(),
c_constraints=[],
st_constraints=st_constraints)
print('calculating MCR-ALS')
mcrar.fit(D, ST=ST_guess, verbose=True)
np.savetxt('out/wavelengths.txt', x)
np.savetxt('out/C-opt.txt', mcrar.C_opt_)
if dct['smooth spectra'] == 'True':
args = (int(dct['smoothing window length']), str(dct['smoothing window']))
S = np.apply_along_axis(smooth, 1, mcrar.ST_opt_, *args).T
np.savetxt('out/S-opt.txt', S)
else:
S = mcrar.ST_opt_.T
np.savetxt('out/S-opt.txt', S)
def test_mcr_ideal_str_regressors(dataset):
""" Test MCR with string-provded regressors"""
C_known, D_known, St_known = dataset
mcrar = McrAR(c_regr='OLS', st_regr='OLS')
mcrar.fit(D_known, ST=St_known, verbose=True)
assert_equal(1, mcrar.n_iter_opt)
assert isinstance(mcrar.c_regressor, pymcr.regressors.OLS)
assert isinstance(mcrar.st_regressor, pymcr.regressors.OLS)
mcrar = McrAR(c_regr='NNLS', st_regr='NNLS')
mcrar.fit(D_known, ST=St_known)
assert_equal(1, mcrar.n_iter_opt)
assert isinstance(mcrar.c_regressor, pymcr.regressors.NNLS)
assert isinstance(mcrar.st_regressor, pymcr.regressors.NNLS)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
# Provided C_known this time
mcrar = McrAR(c_regr='OLS', st_regr='OLS')
mcrar.fit(D_known, C=C_known)
# Turns out some systems get it in 1 iteration, some in 2
# assert_equal(1, mcrar.n_iter_opt)
assert_equal(True, mcrar.n_iter_opt <= 2)
assert ((mcrar.D_ - D_known)**2).mean() < 1e-10
assert ((mcrar.D_opt_ - D_known)**2).mean() < 1e-10
BSS Removal """ tic = time.time() Xd = auxiliary_funs.npass_SGderivative(Xin,1,7,2) #used in pls later #Xd=Xin # Compute ICA ica = FastICA(n_components=nica,tol=1e-8,max_iter=500) ica.fit_transform(Xd.T) # Reconstruct signals, needs the transpose of the matrix Aica = ica.mixing_ # Get estimated miXeg matrix S0ica = (np.linalg.pinv(Aica)@Xin) # Reconstruct signals, needs the transpose of the matrix # Compute MCR mcrals = McrAR(st_regr='NNLS',c_regr=ElasticNet(alpha=1e-4,l1_ratio=0.75),tol_increase=5,tol_n_above_min=500,max_iter=2000,tol_err_change=Xin.mean()*1e-8,c_constraints=[ConstraintNonneg()]) mcrals.fit(Xin, ST= S0ica**2 ) S0mcr = mcrals.ST_opt_; Amcr = mcrals.C_opt_; toc = time.time() runningtime_BSS = toc-tic # How long the decomposition took # Species Identification Cor = auxiliary_funs.correlation(S0mcr,Lnew_h2o,nica).T # with cut-off value # Ivalues = Cor.max(0);#maximum correlation from each column # I = np.where(Ivalues<0.70)[0]; #background values
def test_mcr_errors():
# Providing both C and S^T estimates without C_fix and St_fix
with pytest.raises(TypeError):
mcrar = McrAR()
mcrar.fit(np.random.randn(10, 5),
C=np.random.randn(10, 3),
ST=np.random.randn(3, 5))
# Providing both C and S^T estimates without both C_fix and St_fix
with pytest.raises(TypeError):
mcrar = McrAR()
# Only c_fix
mcrar.fit(np.random.randn(10, 5),
C=np.random.randn(10, 3),
ST=np.random.randn(3, 5),
c_fix=[0])
with pytest.raises(TypeError):
mcrar = McrAR()
# Only st_fix
mcrar.fit(np.random.randn(10, 5),
C=np.random.randn(10, 3),
ST=np.random.randn(3, 5),
st_fix=[0])
# Providing no estimates
with pytest.raises(TypeError):
mcrar = McrAR()
mcrar.fit(np.random.randn(10, 5))
# Unknown regression method
with pytest.raises(ValueError):
mcrar = McrAR(c_regr='NOTREAL')
# regression object with no fit method
with pytest.raises(ValueError):
mcrar = McrAR(c_regr=print)
class FactorizationWidget(QSplitter):
sigPCA = Signal(object)
def __init__(self, headermodel, selectionmodel):
super(FactorizationWidget, self).__init__()
self.headermodel = headermodel
self.selectionmodel = selectionmodel
self.selectionmodel.selectionChanged.connect(self.updateMap)
self.selectionmodel.selectionChanged.connect(self.updateRoiMask)
self.selectMapIdx = 0
self.rightsplitter = QSplitter()
self.rightsplitter.setOrientation(Qt.Vertical)
self.gridwidget = QWidget()
self.gridlayout = QGridLayout()
self.gridwidget.setLayout(self.gridlayout)
self.display = QSplitter()
# self.componentSpectra = PlotWidget()
self.componentSpectra = ComponentPlotWidget()
self._plotLegends = self.componentSpectra.addLegend()
self._colors = ['r', 'g', 'm', 'y', 'c', 'b', 'w'] # color for plots
# self.spectraROI = PlotWidget()
self.NWimage = SlimImageView()
self.NEimage = SlimImageView()
self.SWimage = SlimImageView()
self.SEimage = SlimImageView()
# setup ROI item
sideLen = 10
self.roiList = []
self.maskList = []
self.selectMaskList = []
self._imageDict = {
0: 'NWimage',
1: 'NEimage',
2: 'SWimage',
3: 'SEimage'
}
for i in range(4):
getattr(self, self._imageDict[i]).setPredefinedGradient("viridis")
getattr(self,
self._imageDict[i]).getHistogramWidget().setMinimumWidth(5)
getattr(self, self._imageDict[i]).view.invertY(True)
getattr(
self,
self._imageDict[i]).imageItem.setOpts(axisOrder="row-major")
# set up roi item
roi = PolyLineROI(positions=[[0, 0], [sideLen, 0],
[sideLen, sideLen], [0, sideLen]],
closed=True)
roi.hide()
self.roiInitState = roi.getState()
self.roiList.append(roi)
# set up mask item
maskItem = ImageItem(np.ones((1, 1)),
axisOrder="row-major",
autoLevels=True,
opacity=0.3)
maskItem.hide()
self.maskList.append(maskItem)
# set up select mask item
selectMaskItem = ImageItem(np.ones((1, 1)),
axisOrder="row-major",
autoLevels=True,
opacity=0.3,
lut=np.array([[0, 0, 0], [255, 0, 0]]))
selectMaskItem.hide()
self.selectMaskList.append(selectMaskItem)
# set up image title
getattr(self, self._imageDict[i]).imageTitle = TextItem()
getattr(self, self._imageDict[i]).view.addItem(roi)
getattr(self, self._imageDict[i]).view.addItem(maskItem)
getattr(self, self._imageDict[i]).view.addItem(selectMaskItem)
getattr(self, self._imageDict[i]).view.addItem(
getattr(self, self._imageDict[i]).imageTitle)
self.parametertree = FactorizationParameters(headermodel,
selectionmodel)
self.parameter = self.parametertree.parameter
for i in range(4):
self.parameter.child(
f'Map {i + 1} Component').sigValueChanged.connect(
partial(self.updateComponents, i))
self.addWidget(self.display)
self.addWidget(self.rightsplitter)
self.display.addWidget(self.gridwidget)
self.display.addWidget(self.componentSpectra)
# self.display.addWidget(self.spectraROI)
self.gridlayout.addWidget(self.NWimage, 0, 0, 1, 1)
self.gridlayout.addWidget(self.NEimage, 0, 1, 1, 1)
self.gridlayout.addWidget(self.SWimage, 1, 0, 1, 1)
self.gridlayout.addWidget(self.SEimage, 1, 1, 1, 1)
self.setOrientation(Qt.Horizontal)
self.display.setOrientation(Qt.Vertical)
# buttons layout
self.buttons = QWidget()
self.buttonlayout = QGridLayout()
self.buttons.setLayout(self.buttonlayout)
# set up buttons
self.fontSize = 12
font = QFont("Helvetica [Cronyx]", self.fontSize)
self.computeBtn = QPushButton()
self.computeBtn.setText('Decompose')
self.computeBtn.setFont(font)
self.saveBtn = QPushButton()
self.saveBtn.setText('Save Results')
self.saveBtn.setFont(font)
# add all buttons
self.buttonlayout.addWidget(self.computeBtn)
self.buttonlayout.addWidget(self.saveBtn)
# Headers listview
self.headerlistview = QListView()
self.headerlistview.setModel(headermodel)
self.headerlistview.setSelectionModel(
selectionmodel) # This might do weird things in the map view?
self.headerlistview.setSelectionMode(QListView.SingleSelection)
# add title to list view
self.fontSize = 12
font = QFont("Helvetica [Cronyx]", self.fontSize)
self.mapListWidget = QWidget()
self.listLayout = QVBoxLayout()
self.mapListWidget.setLayout(self.listLayout)
mapListTitle = QLabel('Maps list')
mapListTitle.setFont(font)
self.listLayout.addWidget(mapListTitle)
self.listLayout.addWidget(self.headerlistview)
# adjust right splitter
self.rightsplitter.addWidget(self.parametertree)
self.rightsplitter.addWidget(self.buttons)
self.rightsplitter.addWidget(self.mapListWidget)
self.rightsplitter.setSizes([300, 50, 50])
#connect signals
self.computeBtn.clicked.connect(self.calculate)
self.saveBtn.clicked.connect(self.saveResults)
self.sigPCA.connect(self.showComponents)
def updateRoiMask(self):
if self.selectionmodel.hasSelection():
self.selectMapIdx = self.selectionmodel.selectedIndexes()[0].row()
elif self.headermodel.rowCount() > 0:
self.selectMapIdx = 0
else:
return
# update roi
try:
roiState = self.headermodel.item(self.selectMapIdx).roiState
for i in range(4):
if roiState[0]: #roi on
self.roiList[i].show()
else:
self.roiList[i].hide()
# update roi state
self.roiList[i].blockSignals(True)
self.roiList[i].setState(roiState[1])
self.roiList[i].blockSignals(False)
except Exception:
for i in range(4):
self.roiList[i].hide()
# update automask
try:
maskState = self.headermodel.item(self.selectMapIdx).maskState
for i in range(4):
self.maskList[i].setImage(maskState[1])
if maskState[0]: # automask on
self.maskList[i].show()
else:
self.maskList[i].hide()
except Exception:
pass
# update selectMask
try:
selectMaskState = self.headermodel.item(
self.selectMapIdx).selectState
for i in range(4):
self.selectMaskList[i].setImage(selectMaskState[1])
if selectMaskState[0]: # selectmask on
self.selectMaskList[i].show()
else:
self.selectMaskList[i].hide()
except Exception:
pass
def updateComponents(self, i):
# i is imageview/window number
# component_index is the PCA component index
component_index = self.parameter[f'Map {i + 1} Component']
# update scoreplots on view i
if hasattr(self, '_data_fac') and (self._data_fac is not None):
# update map
self.drawMap(component_index, i)
# update PCA components
if hasattr(self, '_plots'):
# update plots
name = self.parameter['Method'] + str(component_index)
self._plots[i].setData(self.wavenumbers,
self._fac.components_[component_index -
1, :],
name=name)
# update legend label
sample, label = self._plotLegends.items[i]
label.setText(name)
def updateMap(self):
if self.selectionmodel.hasSelection():
self.selectMapIdx = self.selectionmodel.selectedIndexes()[0].row()
elif self.headermodel.rowCount() > 0:
self.selectMapIdx = 0
else:
return
if hasattr(self, '_data_fac') and (self._data_fac is not None):
if len(
self._dataRowSplit
) < self.selectMapIdx + 2: # some maps are not included in the factorization calculation
msg.logMessage(
'One or more maps are not included in the factorization dataset. Please click "calculate" to re-compute factors.',
msg.ERROR)
else:
for i in range(4):
component_index = self.parameter[f'Map {i + 1} Component']
# update map
self.drawMap(component_index, i)
elif hasattr(self, 'imgShapes') and (self.selectMapIdx < len(
self.imgShapes)): #clear maps
for i in range(4):
img = np.zeros((self.imgShapes[self.selectMapIdx][0],
self.imgShapes[self.selectMapIdx][1]))
getattr(self, self._imageDict[i]).setImage(img=img)
def showComponents(self, fac_obj):
# get map ROI selected region
self.selectedPixelsList = [
self.headermodel.item(i).selectedPixels
for i in range(self.headermodel.rowCount())
]
# clear plots and legends
self.componentSpectra.getViewBox().clear()
self.componentSpectra.ymax = 0
for sample, label in self._plotLegends.items[:]:
self._plotLegends.removeItem(label.text)
self.wavenumbers, self._fac, self._data_fac, self._dataRowSplit = fac_obj[
0], fac_obj[1], fac_obj[2], fac_obj[3]
if self._fac is not None:
self._plots = []
for i in range(4):
component_index = self.parameter[f'Map {i + 1} Component']
name = self.parameter['Method'] + str(component_index)
# show loading plots
tmp = self.componentSpectra.plot(
self.wavenumbers,
self._fac.components_[component_index - 1, :],
name=name,
pen=mkPen(self._colors[i], width=2))
tmp.curve.setClickable(True)
tmp.curve.sigClicked.connect(partial(self.curveHighLight, i))
self._plots.append(tmp)
# show score plots
self.drawMap(component_index, i)
# update the last image and loading plots as a recalculation complete signal
N = self.parameter['Components']
self.parameter.child(f'Map 4 Component').setValue(N)
# clear maps
else:
tab_idx = self.headermodel.rowCount() - 1
if tab_idx >= 0:
for i in range(4):
img = np.zeros((self.imgShapes[tab_idx][0],
self.imgShapes[tab_idx][1]))
getattr(self, self._imageDict[i]).setImage(img=img)
def drawMap(self, component_index, i):
# i is imageview/window number
data_slice = self._data_fac[self._dataRowSplit[self.selectMapIdx]:self.
_dataRowSplit[self.selectMapIdx + 1],
component_index - 1]
# draw map
if self.selectedPixelsList[self.selectMapIdx] is None: # full map
img = data_slice.reshape(self.imgShapes[self.selectMapIdx][0],
self.imgShapes[self.selectMapIdx][1])
elif self.selectedPixelsList[self.selectMapIdx].size == 0: # empty ROI
img = np.zeros((self.imgShapes[self.selectMapIdx][0],
self.imgShapes[self.selectMapIdx][1]))
else:
img = np.zeros((self.imgShapes[self.selectMapIdx][0],
self.imgShapes[self.selectMapIdx][1]))
img[self.selectedPixelsList[self.selectMapIdx][:, 0],
self.selectedPixelsList[self.selectMapIdx][:, 1]] = data_slice
img = np.flipud(img)
getattr(self, self._imageDict[i]).setImage(img=img)
# set imageTitle
imageTitle = getattr(self, self._imageDict[i]).imageTitle
title = self.parameter['Method'] + str(component_index)
imageTitle.setHtml(
f'<div style="text-align: center"><span style="color: #FFF; font-size: 8pt">{title}</div>'
)
imageTitle.setPos(0, -5)
def curveHighLight(self, k):
for i in range(4):
if i == k:
self._plots[i].setPen(mkPen(self._colors[k], width=6))
self._plots[i].setZValue(50)
else:
self._plots[i].setPen(mkPen(self._colors[i], width=2))
self._plots[i].setZValue(0)
self.componentSpectra._x, self.componentSpectra._y = self._plots[
k].getData()
self.componentSpectra.getEnergy()
def setHeader(self, field: str):
self.headers = [
self.headermodel.item(i).header
for i in range(self.headermodel.rowCount())
]
self.field = field
wavenum_align = []
self.imgShapes = []
self.rc2indList = []
self.ind2rcList = []
self._dataSets = {'spectra': [], 'volume': []}
# get wavenumbers, imgShapes
for header in self.headers:
dataEvent = next(header.events(fields=[field]))
self.wavenumbers = dataEvent['wavenumbers']
self.N_w = len(self.wavenumbers)
wavenum_align.append(
(round(self.wavenumbers[0]),
self.N_w)) # append (first wavenum value, wavenum length)
self.imgShapes.append(dataEvent['imgShape'])
self.rc2indList.append(dataEvent['rc_index'])
self.ind2rcList.append(dataEvent['index_rc'])
# load data
data = None
try: # spectra datasets
data = header.meta_array('spectra')
except IndexError:
msg.logMessage(
'Header object contained no frames with field '
'{field}'
'.', msg.ERROR)
if data is not None:
self._dataSets['spectra'].append(data)
# NMF path sets
volumeEvent = next(header.events(fields=['volume']))
path = volumeEvent['path'] # readin filepath
self._dataSets['volume'].append(path)
# init maps
if len(self.imgShapes) > 0:
self.showComponents((self.wavenumbers, None, None, None))
if wavenum_align and (wavenum_align.count(wavenum_align[0]) !=
len(wavenum_align)):
MsgBox(
'Length of wavenumber arrays of displayed maps are not equal. \n'
'Perform PCA or NMF on these maps will lead to error.', 'warn')
# self.parametertree.setHeader(self.wavenumbers, self.imgShapes, self.rc2indList, self.ind2rcList)
def calculate(self):
N = self.parameter['Components']
#set decompose method
if self.parameter['Method'] == 'PCA':
self.method = 'PCA'
self.field = 'spectra'
elif self.parameter['Method'] == 'NMF':
self.method = 'NMF'
self.field = 'volume'
elif self.parameter['Method'] == 'MCR':
self.method = 'MCR'
self.field = 'spectra'
if hasattr(self, '_dataSets'):
wavROIList = []
for entry in self.parameter['Wavenumber Range'].split(','):
try:
wavROIList.append(val2ind(int(entry), self.wavenumbers))
except:
continue
# Select wavenumber region
if len(wavROIList) % 2 == 0:
wavROIList = sorted(wavROIList)
wavROIidx = []
for i in range(len(wavROIList) // 2):
wavROIidx += list(
range(wavROIList[2 * i], wavROIList[2 * i + 1] + 1))
else:
msg.logMessage('"Wavenumber Range" values must be in pairs',
msg.ERROR)
MsgBox('Factorization computation aborted.', 'error')
return
self.wavenumbers_select = self.wavenumbers[wavROIidx]
# get map ROI selected region
self.selectedPixelsList = [
self.headermodel.item(i).selectedPixels
for i in range(self.headermodel.rowCount())
]
self.df_row_idx = [] # row index for dataframe data_fac
msg.showMessage('Start computing', self.method + '. Image shape:',
str(self.imgShapes))
self.dataRowSplit = [
0
] # remember the starting/end row positions of each dataset
if self.field == 'spectra': # PCA workflow
self.N_w = len(self.wavenumbers_select)
self._allData = np.empty((0, self.N_w))
for i, data in enumerate(
self._dataSets['spectra']): # i: map idx
if self.selectedPixelsList[i] is None:
n_spectra = len(data)
tmp = np.zeros((n_spectra, self.N_w))
for j in range(n_spectra):
tmp[j, :] = data[j][wavROIidx]
self.df_row_idx.append((self.ind2rcList[i][j], j))
else:
n_spectra = len(self.selectedPixelsList[i])
tmp = np.zeros((n_spectra, self.N_w))
for j in range(n_spectra): # j: jth selected pixel
row_col = tuple(self.selectedPixelsList[i][j])
tmp[j, :] = data[self.rc2indList[i]
[row_col]][wavROIidx]
self.df_row_idx.append(
(row_col, self.rc2indList[i][row_col]))
self.dataRowSplit.append(self.dataRowSplit[-1] + n_spectra)
self._allData = np.append(self._allData, tmp, axis=0)
if len(self._allData) > 0:
if self.method == 'PCA':
self.data_fac_name = 'data_PCA' # define pop up plots labels
# normalize and mean center
if self.parameter[
'Normalization'] == 'L1': # normalize
data_norm = Normalizer(norm='l1').fit_transform(
self._allData)
elif self.parameter['Normalization'] == 'L2':
data_norm = Normalizer(norm='l2').fit_transform(
self._allData)
else:
data_norm = self._allData
#subtract mean
data_centered = StandardScaler(
with_std=False).fit_transform(data_norm)
# Do PCA
self.PCA = PCA(n_components=N)
self.PCA.fit(data_centered)
self.data_PCA = self.PCA.transform(data_centered)
# pop up plots
self.popup_plots()
elif self.method == 'MCR':
self.data_fac_name = 'data_MCR' # define pop up plots labels
# Do ICA to find initial estimate of ST matrix
self.ICA = FastICA(n_components=N)
self.ICA.fit(self._allData)
# Do MCR
self.MCR = McrAR(max_iter=100,
c_regr=self.parameter['C regressor'],
st_regr='NNLS',
tol_err_change=1e-6,
tol_increase=0.5)
self.MCR.fit(self._allData, ST=self.ICA.components_)
self.MCR.components_ = self.MCR.ST_opt_
self.data_MCR = self.MCR.C_opt_
#test ICA
# self.MCR = self.ICA
# self.data_MCR = self.ICA.transform(self._allData)
# pop up plots
self.popup_plots()
else:
msg.logMessage(
'The data matrix is empty. No PCA is performed.',
msg.ERROR)
MsgBox('The data matrix is empty. No PCA is performed.',
'error')
self.PCA, self.data_PCA = None, None
self.MCR, self.data_MCR = None, None
# emit PCA and transformed data
if self.method == 'PCA':
self.sigPCA.emit((self.wavenumbers_select, self.PCA,
self.data_PCA, self.dataRowSplit))
elif self.method == 'MCR':
self.sigPCA.emit((self.wavenumbers_select, self.MCR,
self.data_MCR, self.dataRowSplit))
elif self.field == 'volume': # NMF workflow
data_files = []
wav_masks = []
row_idx = np.array([], dtype='int')
self.allDataRowSplit = [0] # row split for complete datasets
for i, file in enumerate(self._dataSets['volume']):
ir_data, fmt = read_map.read_all_formats(file)
n_spectra = ir_data.data.shape[0]
self.allDataRowSplit.append(self.allDataRowSplit[-1] +
n_spectra)
data_files.append(ir_data)
ds = data_prep.data_prepper(ir_data)
wav_masks.append(ds.decent_bands)
# row selection
if self.selectedPixelsList[i] is None:
row_idx = np.append(
row_idx,
np.arange(self.allDataRowSplit[-2],
self.allDataRowSplit[-1]))
for k, v in self.rc2indList[i].items():
self.df_row_idx.append((k, v))
else:
n_spectra = len(self.selectedPixelsList[i])
for j in range(n_spectra):
row_col = tuple(self.selectedPixelsList[i][j])
row_idx = np.append(
row_idx, self.allDataRowSplit[-2] +
self.rc2indList[i][row_col])
self.df_row_idx.append(
(row_col, self.rc2indList[i][row_col]))
self.dataRowSplit.append(
self.dataRowSplit[-1] +
n_spectra) # row split for ROI selected rows
# define pop up plots labels
self.data_fac_name = 'data_NMF'
if len(self.df_row_idx) > 0:
# aggregate datasets
ir_data_agg = aggregate_data(self._dataSets['volume'],
data_files, wav_masks)
col_idx = list(
set(wavROIidx) & set(ir_data_agg.master_wmask))
self.wavenumbers_select = self.wavenumbers[col_idx]
ir_data_agg.data = ir_data_agg.data[:, col_idx]
ir_data_agg.data = ir_data_agg.data[row_idx, :]
# perform NMF
self.NMF = NMF(n_components=N)
self.data_NMF = self.NMF.fit_transform(ir_data_agg.data)
# pop up plots
self.popup_plots()
else:
msg.logMessage(
'The data matrix is empty. No NMF is performed.',
msg.ERROR)
MsgBox('The data matrix is empty. No NMF is performed.',
'error')
self.NMF, self.data_NMF = None, None
# emit NMF and transformed data : data_NMF
self.sigPCA.emit((self.wavenumbers_select, self.NMF,
self.data_NMF, self.dataRowSplit))
def popup_plots(self):
# component variance ratio plot
if self.method == 'PCA':
plt.plot(
getattr(self, self.method).explained_variance_ratio_, 'o-b')
ax = plt.gca()
ax.set_ylabel('Explained variance ratio', fontsize=16)
ax.set_xlabel('Component number', fontsize=16)
ax.set_xticks(np.arange(self.parameter['Components']))
# loadings plot
plt.figure()
labels = []
for i in range(getattr(self, self.method).components_.shape[0]):
labels.append(self.method + str(i + 1))
plt.plot(self.wavenumbers_select,
getattr(self, self.method).components_[i, :],
'-',
label=labels[i])
loadings_legend = plt.legend(loc='best')
plt.setp(loadings_legend, draggable=True)
plt.xlim([max(self.wavenumbers_select), min(self.wavenumbers_select)])
ax = plt.gca()
ax.set_xlabel('Wavenumber$(cm^{-1})$', fontsize=16)
# matrix plot
groupLabel = np.zeros((self.dataRowSplit[-1], 1))
for i in range(len(self.dataRowSplit) - 1):
groupLabel[self.dataRowSplit[i]:self.dataRowSplit[i + 1]] = int(i)
df_scores = pd.DataFrame(np.append(getattr(self, self.data_fac_name),
groupLabel,
axis=1),
columns=labels + ['Group label'])
grid = sns.pairplot(df_scores, vars=labels, hue="Group label")
# change legend properties
legend_labels = []
for i in range(self.headermodel.rowCount()):
if (self.selectedPixelsList[i] is
None) or (self.selectedPixelsList[i].size > 0):
legend_labels.append(self.headermodel.item(i).data(0))
for t, l in zip(grid._legend.texts, legend_labels):
t.set_text(l)
plt.setp(grid._legend.get_texts(), fontsize=14)
plt.setp(grid._legend.get_title(), fontsize=14)
plt.setp(grid._legend,
bbox_to_anchor=(0.2, 0.95),
frame_on=True,
draggable=True)
plt.setp(grid._legend.get_frame(), edgecolor='k', linewidth=1, alpha=1)
plt.show()
def saveResults(self):
if (hasattr(self, 'PCA') and self.PCA is not None) or (hasattr(self, 'NMF') and self.NMF is not None)\
or (hasattr(self, 'MCR') and self.MCR is not None):
name = self.method
df_fac_components = pd.DataFrame(getattr(self, name).components_,
columns=self.wavenumbers_select)
df_data_fac = pd.DataFrame(getattr(self, self.data_fac_name),
index=self.df_row_idx)
df_fac_components.to_csv(name + '_components.csv')
df_data_fac.to_csv(name + '_data.csv')
np.savetxt(name + '_mapRowSplit.csv',
np.array(self.dataRowSplit),
fmt='%d',
delimiter=',')
MsgBox(name + ' components successfully saved!')
else:
MsgBox('No factorization components available.')
def calculate(self):
N = self.parameter['Components']
#set decompose method
if self.parameter['Method'] == 'PCA':
self.method = 'PCA'
self.field = 'spectra'
elif self.parameter['Method'] == 'NMF':
self.method = 'NMF'
self.field = 'volume'
elif self.parameter['Method'] == 'MCR':
self.method = 'MCR'
self.field = 'spectra'
if hasattr(self, '_dataSets'):
wavROIList = []
for entry in self.parameter['Wavenumber Range'].split(','):
try:
wavROIList.append(val2ind(int(entry), self.wavenumbers))
except:
continue
# Select wavenumber region
if len(wavROIList) % 2 == 0:
wavROIList = sorted(wavROIList)
wavROIidx = []
for i in range(len(wavROIList) // 2):
wavROIidx += list(
range(wavROIList[2 * i], wavROIList[2 * i + 1] + 1))
else:
msg.logMessage('"Wavenumber Range" values must be in pairs',
msg.ERROR)
MsgBox('Factorization computation aborted.', 'error')
return
self.wavenumbers_select = self.wavenumbers[wavROIidx]
# get map ROI selected region
self.selectedPixelsList = [
self.headermodel.item(i).selectedPixels
for i in range(self.headermodel.rowCount())
]
self.df_row_idx = [] # row index for dataframe data_fac
msg.showMessage('Start computing', self.method + '. Image shape:',
str(self.imgShapes))
self.dataRowSplit = [
0
] # remember the starting/end row positions of each dataset
if self.field == 'spectra': # PCA workflow
self.N_w = len(self.wavenumbers_select)
self._allData = np.empty((0, self.N_w))
for i, data in enumerate(
self._dataSets['spectra']): # i: map idx
if self.selectedPixelsList[i] is None:
n_spectra = len(data)
tmp = np.zeros((n_spectra, self.N_w))
for j in range(n_spectra):
tmp[j, :] = data[j][wavROIidx]
self.df_row_idx.append((self.ind2rcList[i][j], j))
else:
n_spectra = len(self.selectedPixelsList[i])
tmp = np.zeros((n_spectra, self.N_w))
for j in range(n_spectra): # j: jth selected pixel
row_col = tuple(self.selectedPixelsList[i][j])
tmp[j, :] = data[self.rc2indList[i]
[row_col]][wavROIidx]
self.df_row_idx.append(
(row_col, self.rc2indList[i][row_col]))
self.dataRowSplit.append(self.dataRowSplit[-1] + n_spectra)
self._allData = np.append(self._allData, tmp, axis=0)
if len(self._allData) > 0:
if self.method == 'PCA':
self.data_fac_name = 'data_PCA' # define pop up plots labels
# normalize and mean center
if self.parameter[
'Normalization'] == 'L1': # normalize
data_norm = Normalizer(norm='l1').fit_transform(
self._allData)
elif self.parameter['Normalization'] == 'L2':
data_norm = Normalizer(norm='l2').fit_transform(
self._allData)
else:
data_norm = self._allData
#subtract mean
data_centered = StandardScaler(
with_std=False).fit_transform(data_norm)
# Do PCA
self.PCA = PCA(n_components=N)
self.PCA.fit(data_centered)
self.data_PCA = self.PCA.transform(data_centered)
# pop up plots
self.popup_plots()
elif self.method == 'MCR':
self.data_fac_name = 'data_MCR' # define pop up plots labels
# Do ICA to find initial estimate of ST matrix
self.ICA = FastICA(n_components=N)
self.ICA.fit(self._allData)
# Do MCR
self.MCR = McrAR(max_iter=100,
c_regr=self.parameter['C regressor'],
st_regr='NNLS',
tol_err_change=1e-6,
tol_increase=0.5)
self.MCR.fit(self._allData, ST=self.ICA.components_)
self.MCR.components_ = self.MCR.ST_opt_
self.data_MCR = self.MCR.C_opt_
#test ICA
# self.MCR = self.ICA
# self.data_MCR = self.ICA.transform(self._allData)
# pop up plots
self.popup_plots()
else:
msg.logMessage(
'The data matrix is empty. No PCA is performed.',
msg.ERROR)
MsgBox('The data matrix is empty. No PCA is performed.',
'error')
self.PCA, self.data_PCA = None, None
self.MCR, self.data_MCR = None, None
# emit PCA and transformed data
if self.method == 'PCA':
self.sigPCA.emit((self.wavenumbers_select, self.PCA,
self.data_PCA, self.dataRowSplit))
elif self.method == 'MCR':
self.sigPCA.emit((self.wavenumbers_select, self.MCR,
self.data_MCR, self.dataRowSplit))
elif self.field == 'volume': # NMF workflow
data_files = []
wav_masks = []
row_idx = np.array([], dtype='int')
self.allDataRowSplit = [0] # row split for complete datasets
for i, file in enumerate(self._dataSets['volume']):
ir_data, fmt = read_map.read_all_formats(file)
n_spectra = ir_data.data.shape[0]
self.allDataRowSplit.append(self.allDataRowSplit[-1] +
n_spectra)
data_files.append(ir_data)
ds = data_prep.data_prepper(ir_data)
wav_masks.append(ds.decent_bands)
# row selection
if self.selectedPixelsList[i] is None:
row_idx = np.append(
row_idx,
np.arange(self.allDataRowSplit[-2],
self.allDataRowSplit[-1]))
for k, v in self.rc2indList[i].items():
self.df_row_idx.append((k, v))
else:
n_spectra = len(self.selectedPixelsList[i])
for j in range(n_spectra):
row_col = tuple(self.selectedPixelsList[i][j])
row_idx = np.append(
row_idx, self.allDataRowSplit[-2] +
self.rc2indList[i][row_col])
self.df_row_idx.append(
(row_col, self.rc2indList[i][row_col]))
self.dataRowSplit.append(
self.dataRowSplit[-1] +
n_spectra) # row split for ROI selected rows
# define pop up plots labels
self.data_fac_name = 'data_NMF'
if len(self.df_row_idx) > 0:
# aggregate datasets
ir_data_agg = aggregate_data(self._dataSets['volume'],
data_files, wav_masks)
col_idx = list(
set(wavROIidx) & set(ir_data_agg.master_wmask))
self.wavenumbers_select = self.wavenumbers[col_idx]
ir_data_agg.data = ir_data_agg.data[:, col_idx]
ir_data_agg.data = ir_data_agg.data[row_idx, :]
# perform NMF
self.NMF = NMF(n_components=N)
self.data_NMF = self.NMF.fit_transform(ir_data_agg.data)
# pop up plots
self.popup_plots()
else:
msg.logMessage(
'The data matrix is empty. No NMF is performed.',
msg.ERROR)
MsgBox('The data matrix is empty. No NMF is performed.',
'error')
self.NMF, self.data_NMF = None, None
# emit NMF and transformed data : data_NMF
self.sigPCA.emit((self.wavenumbers_select, self.NMF,
self.data_NMF, self.dataRowSplit))
#mcrals.fit(D, C=initial_conc)
Examples to set the different options
In the following, we initiate a McrAls object
with a st_regr setup with NNLS and
c_reg to OLS.
One can select regressor with a string, or can import the class and instanstiate
mcrals = McrAls(max_iter=100, st_regr='NNLS', c_regr=OLS(),
c_constraints=[ConstraintNonneg(), ConstraintNorm()])
"""
tic = time.time()
mcrals = McrAR(c_regr=linear_model.ElasticNet(alpha=1e-5, l1_ratio=0.75),
max_iter=700,
tol_err_change=Xe[index, :].max() * 1e-8,
st_regr='NNLS',
c_constraints=[ConstraintNonneg()])
mcrals.fit(Xe[index, :], ST=S0ica**2)
toc = time.time()
runningtime_mcrals = toc - tic # How long the decomposition took
S0mcr = mcrals.ST_opt_
Amcr = mcrals.C_opt_
Sdmcr = (np.linalg.pinv(Amcr) @ Xd[index, :])
"""
Species Identification
"""
# Comparison with Library
Cor = auxiliary_funs.correlation(S0mcr, sources, nica)
I = Cor.argmax(1)
