# specs = cut_specs(specs, 4100, 5500)
# %%
X, y, wave_number, ref = importLuzCol(specs, lab, 4)
# split dataset in train and test data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.3, random_state=42
)
# %%
# transformation pipeline
# scale y
y_scaler = GlobalStandardScaler()
y_train = y_scaler.fit_transform(y_train)
y_test = y_scaler.transform(y_test)
pip_dev0 = Pipeline(
[
("scaleing_X", GlobalStandardScaler()),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=0)),
("variable_selection", EnetSelect()),
]
)
# to perform variable selection y values are corrleated in the fit method
X_train_0 = pip_dev0.fit_transform(X_train, y_train)
X_test_0 = pip_dev0.transform(X_test)
# %%
X, y, wave_number, ref = importLuzCol(specs, lab, 4)
# split dataset in train and test data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.1, random_state=42
)
# %%
dev_0 = Pipeline(
[
("scaleing_X", GlobalStandardScaler(with_std=False)),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=0)),
]
)
X_0 = dev_0.fit_transform(X)
# %%
dev_1 = Pipeline(
[
("scaleing_X", GlobalStandardScaler(with_std=False)),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=1)),
X, y, wave_number, feat_ref = importLuzCol(specs, lab, 4)
# split dataset in train and test data
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
# %%
plot_spec(wave_number, X)
# %%
# scale y
y_scaler = GlobalStandardScaler()
y_train = y_scaler.fit_transform(y_train)
y_test = y_scaler.transform(y_test)
dev_0 = Pipeline([("scaleing_X", GlobalStandardScaler()),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=1))])
X_train_0 = dev_0.fit_transform(X_train)
X_test_0 = dev_0.fit_transform(X_test)
plot_spec(wave_number, X_train_0)
# %%
pls_opt = PLSOptimizer()
pls_opt.fit(X_train_0, y_train, max_comp=20)
pls_opt.plot(wave_number, X_train_0)
# specs = pd.read_csv('./luzrawSpectra/nirMatrix.csv') # cut spectra
specs = pd.read_csv("/Users/maxprem/nirPy/calData_full.csv") # full spectra
lab = pd.read_excel("/Users/maxprem/nirGit/nirpy/luzrawSpectra/labData.xlsx")
# input wavenumber to cut spectra
specs = cut_specs(specs, 4100, 5500)
X, y, wave_number, ref = importLuzCol(specs, lab, 4)
# %%
x1 = X[0:9, :]
plot_spec(wave_number, x1)
# %%
aug_pipline = Pipeline([("scaleing_X", GlobalStandardScaler()),
("dataaugmentation", Dataaugument()),
("scatter_correction", EmscScaler())])
# to perform variable selection y values are corrleated in the fit met
x_aug = aug_pipline.fit_transform(x1)
plot_spec(wave_number, x_aug)
emsc = EmscScaler()
x1_emsc = emsc.fit_transform(x_aug)
x10_emsc = emsc.fit_transform(x1)
plot_spec(wave_number, x1_emsc)
plot_spec(wave_number, x10_emsc)
X, y, wave_number, feat_ref = importLuzCol(specs, lab, 4)
# split dataset in train and test data
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
# %%
plot_spec(wave_number, X)
# %%
dev_0 = Pipeline([("scaleing_X", GlobalStandardScaler()),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=0))])
X_0 = dev_0.fit_transform(X)
plot_spec(wave_number, X_0)
# %%
dev_1 = Pipeline([("scaleing_X", GlobalStandardScaler()),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=1))])
X_1 = dev_1.fit_transform(X)
plot_spec(wave_number, X_1)
# %%
X, y, wave_number, ref = importLuzCol(specs, lab, 2)
# split dataset in train and test data
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
# %%
# transformation pipeline
# scale y
y_scaler = GlobalStandardScaler()
y_train = y_scaler.fit_transform(y_train)
y_test = y_scaler.transform(y_test)
pip_dev0 = Pipeline([
("scaleing_X", GlobalStandardScaler()),
("scatter_correction", EmscScaler()),
("smmothing", SavgolFilter(polyorder=2, deriv=0)),
])
X_train_0 = pip_dev0.fit_transform(X_train, y_train)
X_test_0 = pip_dev0.transform(X_test)
data_en0 = {"X": X_train_0, "y": y_train, "X_test": X_test_0, "y_test": y_test}
# %%
# variable selection on whole data