def conf_interval_beta(self, y, y_pred, X):
"""Estimates the 99% confidence interval for array of parameters beta.
Parameters:
-----------
y: 1-dimensional array
Ground truth dependent variable.
y_pred: 1-dimensional array
Predicted dependent variable.
X: 2-dimensional array
Design matrix with rows as data points and columns as features.
Returns:
--------
confidence_interval:
list(array, array)
Lowest and highest possible values for beta, with 99% confidence.
confidence_deviation:
array
Deviation from value in confidence interval.
"""
sigma2_y = metrics.MSE(y, y_pred)
#print((sigma2_y * np.linalg.pinv(X)).diagonal())
sigma_beta = np.sqrt((sigma2_y * np.linalg.inv(X.T @ X)).diagonal())
confidence_interval = np.array(
[self.beta - 2 * sigma_beta, self.beta + 2 * sigma_beta])
return confidence_interval, 2 * sigma_beta
def test(self):
"""Tests out the models that validate and optimal_model_search has determined are best.
Uses the pandas.DataFrame optimal_model created by optimal_model_search method, and tests each of the models
on testing data. Plots the result side by side with the actual dependent variable.
"""
for (model_name, technique), (_lambda, poly, _, _) in self.optimal_model.iterrows():
for _model in self.models:
if _model.name == model_name:
model = _model
X = poly_design_matrix(int(poly), self.data['x_train'])
model.scaler.fit(X[:,1:])
X[:,1:] = model.scaler.transform(X[:,1:])
model.set_lambda(_lambda)
model.fit(X, self.data['y_train'])
X_test = poly_design_matrix(int(poly), self.data['x_test'])
X_test[:,1:] = model.scaler.transform(X_test[:,1:])
y_pred = model.predict(X_test)
mse = metrics.MSE(data['y_test'], y_pred)
self.optimal_model.loc[model_name, technique]['Test error'] = mse
if self.verbose >= 1:
print(f"MSE: {mse} for poly = {poly} and lambda = {_lambda} with model {model_name}.")
plotting.side_by_side(
['Predicted', self.data['x_test'], y_pred],
['Ground truth', self.data['x_test'], self.data['y_test']],
title=f"Test: {model.name}, $p$={poly}, $\\lambda$={_lambda}, best according to {technique}",
filename=f"{name}_test_{model.name}_p{poly}{[f'_lambda{_lambda}', ''][int(_lambda==0)]}")
def perform_analysis():
"""
:return:
"""
parser = parse_arg()
args = parser.parse_args()
# if len(sys.argv) == 1: # no arguments, so print help message
# print("""Usage: python script.py data_path program_input out_path""")
# return
#
# dir_in = os.getcwd()
# dir_out = os.getcwd()
#
# try:
# dir_in = sys.argv[1]
# dir_out = sys.argv[2]
# except:
# print("Parameters: path/to/simple/file input/folder output/folder")
# sys.exit(0)
#df = pd.read_csv(args.dir_in)
df = pd.read_csv(args.input)
(cal_df, tst_df) = separating_data_set(df)
(x_train, y_train) = splitting_dataset(cal_df)
(x_test, y_test) = splitting_dataset(tst_df)
print(x_train)
print(y_train)
print(x_test)
print(y_test)
model = build_fnn(x_train)
model.summary()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
random.seed(1)
model.fit(x_train, y_train, batch_size=args.batchSize, epochs=args.epochSize, validation_data=(x_test, y_test), callbacks=[early_stop])
fnn_losses = pd.DataFrame(model.history.history)
create_losses_plot(fnn_losses)
predictions = model.predict(x_test)
print("MSE:", metrics.MSE(y_test, predictions))
print("RMSE:", metrics.RMSE(y_test, predictions))
print("R-square:", metrics.R2(y_test, predictions))
print("RPD:", metrics.RPD(y_test, predictions))
create_prediction_plot(y_test, predictions)
def bootstrap(model, X_train, X_validate, y_train, y_validate, R=50):
"""Resampling techique that draws with replacement from training set, and validates how well a model fitted on that set does.
Parameters:
-----------
model: object
Instance of class with a scaler, a fit method and a predict method.
X_train: 2-dimensional array
Design matrix with rows as data points and columns as features, and training data.
X_validate: 2-dimensional array
Design matrix with rows as data points and columns as features, and validation data.
y_train: 1-dimensional array
Dependent variable, training data.
y_validate: 1-dimensional array
Dependent variable, validate data.
R: int
Number of resamples.
Returns:
--------
mse: int
Mean squared error from validation.
bias: int
Bias for models fitted on the different samples of training data.
var: int
Variation for models fitted on the different samples of training data.
"""
n = X_train.shape[0]
y_boot_tilde = np.empty((R, y_validate.shape[0]))
for r in range(R):
boot_mask = np.random.randint(0, n, n)
X_boot = X_train[boot_mask,:]
y_boot = y_train[boot_mask]
model.scaler.fit(X_boot[:,1:])
X_boot[:,1:] = model.scaler.transform(X_boot[:,1:])
model.fit(X_boot, y_boot)
X_validate[:,1:] = model.scaler.transform(X_validate[:,1:])
y_boot_tilde[r] = model.predict(X_validate)
mse = metrics.MSE(y_validate, y_boot_tilde)
bias = metrics.model_bias(y_validate, y_boot_tilde)
var = metrics.model_variance(y_boot_tilde)
return mse, bias, var
def kfold(model, X, y, n_folds=5):
"""Resampling techique that draws with replacement from training set, and validates how well a model fitted on that set does.
Parameters:
-----------
model: object
Instance of class with a scaler, a fit method and a predict method.
X: 2-dimensional array
Design matrix with rows as data points and columns as features, and both training and validation data.
It will use random parts of it, but only so much that it trains on half of it.
y: 1-dimensional array
Dependent variable, both training and validation data.
n_folds: int
Number of folds the data is divided into.
Returns:
mse: int
Mean squared error from validation.
"""
datasize = X.shape[0]//n_folds
foldsize = datasize//n_folds
shuffled_indexes = np.arange(datasize, dtype=np.int32)
np.random.shuffle(shuffled_indexes)
# To make sure bootstrap and kfold get about as much data in training set, and is shuffled
X_shuffled = X[shuffled_indexes]
y_shuffled = y[shuffled_indexes]
mse = np.empty(n_folds)
for k in range(n_folds):
k_validate = np.s_[int(k*foldsize):int((k+1)*foldsize)]
indexes = np.arange(datasize)
k_train = np.s_[np.logical_or(indexes < k*foldsize, (k+1)*foldsize < indexes)]
model.scaler.fit(X_shuffled[k_train,1:])
X_shuffled[:,1:] = model.scaler.transform(X_shuffled[:,1:])
model.fit(X_shuffled[k_train], y_shuffled[k_train])
y_pred = model.predict(X_shuffled[k_validate])
mse[k] = metrics.MSE(y_shuffled[k_validate], y_pred)
return np.mean(mse)
def main():
# paths to the models
model_paths = [
os.path.join("..", "models", "SRDense-Type-3_ep80.h5"),
os.path.join("..", "models", "srdense-norm.h5"),
os.path.join("..", "models", "srresnet85.h5"),
os.path.join("..", "models", "gen_model90.h5"),
os.path.join("..", "models", "srgan60.h5"),
os.path.join("..", "models", "srgan-mse-20.h5"), "Nearest"
]
# corresponding names of the models
model_names = [
"SRDense", "SRDense-norm", "SRResNet", "SRGAN-from-scratch",
"SRGAN-percept.-loss", "SRGAN-mse", "NearestNeighbor"
]
# corresponding tile shapes
tile_shapes = [((168, 168), (42, 42)), ((168, 168), (42, 42)),
((504, 504), (126, 126)), ((336, 336), (84, 84)),
((504, 504), (126, 126)), ((504, 504), (126, 126)),
((336, 336), (84, 84))]
# used to load the models with custom loss functions
loss = VGG_LOSS((504, 504, 3))
custom_objects = [{}, {
"tf": tf
}, {
"tf": tf
}, {
"tf": tf
}, {
"tf": tf
}, {
"tf": tf
}, {}]
# creating a list of test images
# [(lr, hr)]
DOWN_SCALING_FACTOR = 4
INTERPOLATION = cv2.INTER_CUBIC
test_images = []
root = os.path.join("..", "DSIDS", "test")
# iterating over all files in the test folder
for img in os.listdir(root):
# chekcing if the file is an image
if not ".jpg" in img:
continue
hr = Utils.crop_into_lr_shape(cv2.cvtColor(
cv2.imread(os.path.join(root, img), cv2.IMREAD_COLOR),
cv2.COLOR_BGR2RGB),
shape=(3024, 4032))
lr = cv2.resize(hr, (0, 0),
fx=1 / DOWN_SCALING_FACTOR,
fy=1 / DOWN_SCALING_FACTOR,
interpolation=INTERPOLATION)
test_images.append((lr, hr))
if TILES:
'''
First calculating performance metrics on single image tiles
'''
tile_performance = {}
for i, mp in tqdm(enumerate(model_paths)):
keras.backend.clear_session()
# first step: load the model
if i < 6:
model = load_model(mp, custom_objects=custom_objects[i])
mse = []
psnr = []
ssim = []
mssim = []
# second step: iterate over the test images
for test_pair in tqdm(test_images):
# third step: tile the test image
lr_tiles = Utils.tile_image(test_pair[0],
shape=tile_shapes[i][1])
hr_tiles = Utils.tile_image(test_pair[1],
shape=tile_shapes[i][0])
m = []
p = []
s = []
ms = []
# fourth step: iterate over the tiles
for lr, hr in zip(lr_tiles, hr_tiles):
# fifth step: calculate the sr tile
if i < 2:
if i == 1:
lr = lr.astype(np.float64)
lr = lr / 255
tmp = np.squeeze(
model.predict(np.expand_dims(lr, axis=0)))
if i == 1:
tmp = tmp * 255
tmp[tmp < 0] = 0
tmp[tmp > 255] = 255
sr = tmp.astype(np.uint8)
elif i < 6:
sr = Utils.denormalize(
np.squeeze(model.predict(
np.expand_dims(rescale_imgs_to_neg1_1(lr),
axis=0)),
axis=0))
else:
sr = cv2.resize(lr, (0, 0),
fx=4,
fy=4,
interpolation=cv2.INTER_NEAREST)
# sixth step: append the calculated metric
m.append(metrics.MSE(hr, sr))
p.append(metrics.PSNR(hr, sr))
s.append(metrics.SSIM(hr, sr))
ms.append(metrics.MSSIM(hr, sr))
# seventh step: append the mean metric for this image
mse.append(np.mean(m))
psnr.append(np.mean(p))
ssim.append(np.mean(s))
mssim.append(np.mean(ms))
# eight step: append the mean metric for this model
tile_performance[model_names[i]] = (np.mean(mse), np.mean(psnr),
np.mean(ssim), np.mean(mssim))
# final output
print("Performance on single tiles:")
f = open("tile_performance.txt", "w")
for key in tile_performance:
print(
key + ": MSE = " + str(tile_performance[key][0]) +
", PSNR = " + str(tile_performance[key][1]) + ", SSIM = " +
str(tile_performance[key][2]),
", MSSIM = " + str(tile_performance[key][3]))
f.write(key + " " + str(tile_performance[key][0]) + " " +
str(tile_performance[key][1]) + " " +
str(tile_performance[key][2]) + " " +
str(tile_performance[key][3]) + "\n")
f.close()
if WHOLE_LR:
'''
Second calculating performance metrics on a single upscaled image
'''
img_performance = {}
for i, mp in tqdm(enumerate(model_paths)):
keras.backend.clear_session()
# first step: load the model
if i < 6:
model = load_model(mp, custom_objects=custom_objects[i])
# second step: changing the input layer
_in = Input(shape=test_images[0][0].shape)
_out = model(_in)
_model = Model(_in, _out)
mse = []
psnr = []
ssim = []
mssim = []
# third step: iterate over the test images
for test_pair in tqdm(test_images):
# fourth step: calculate the sr image
try:
if i < 2:
if i == 1:
lr = test_pair[0].astype(np.float64)
lr = lr / 255
else:
lr = test_pair[0]
tmp = np.squeeze(
_model.predict(np.expand_dims(lr, axis=0)))
if i == 1:
tmp = tmp * 255
tmp[tmp < 0] = 0
tmp[tmp > 255] = 255
sr = tmp.astype(np.uint8)
elif i < 6:
sr = Utils.denormalize(
np.squeeze(_model.predict(
np.expand_dims(rescale_imgs_to_neg1_1(
test_pair[0]),
axis=0)),
axis=0))
else:
sr = cv2.resize(test_pair[0], (0, 0),
fx=4,
fy=4,
interpolation=cv2.INTER_NEAREST)
# fifth step: append the metric for this image
mse.append(metrics.MSE(test_pair[1], sr))
psnr.append(metrics.PSNR(test_pair[1], sr))
ssim.append(metrics.SSIM(test_pair[1], sr))
mssim.append(metrics.MSSIM(test_pair[1], sr))
except:
mse.append("err")
psnr.append("err")
ssim.append("err")
mssim.append("err")
# sixth step: append the mean metric for this model
try:
img_performance[model_names[i]] = (np.mean(mse), np.mean(psnr),
np.mean(ssim),
np.mean(mssim))
except:
img_performance[model_names[i]] = ("err", "err", "err", "err")
# final output
print("Performance on whole lr:")
f = open("whole_lr_performance.txt", "w")
for key in img_performance:
print(
key + ": MSE = " + str(img_performance[key][0]) +
", PSNR = " + str(img_performance[key][1]) + ", SSIM = " +
str(img_performance[key][2]),
", MSSIM = " + str(img_performance[key][3]))
f.write(key + " " + str(img_performance[key][0]) + " " +
str(img_performance[key][1]) + " " +
str(img_performance[key][2]) + " " +
str(img_performance[key][3]) + "\n")
f.close()
if STITCHED:
'''
Second calculating performance metrics on a stitched image
'''
stitch_performance = {}
for i, mp in tqdm(enumerate(model_paths)):
keras.backend.clear_session()
# first step: load the model
if i < 6:
model = load_model(mp, custom_objects=custom_objects[i])
mse = []
psnr = []
ssim = []
mssim = []
o_mse = []
o_psnr = []
o_ssim = []
o_mssim = []
# second step: iterate over the test images
for test_pair in tqdm(test_images):
# third step: tile the test image
lr_tiles = Utils.tile_image(test_pair[0],
shape=tile_shapes[i][1])
lr_tiles_overlap = Utils.tile_image(test_pair[0],
shape=tile_shapes[i][1],
overlap=True)
sr_tiles = []
sr_tiles_overlap = []
# fourth step: iterate over the tiles
for lr in lr_tiles:
# fifth step: calculate the sr tiles
if i < 2:
if i == 1:
lr = lr.astype(np.float64)
lr = lr / 255
tmp = np.squeeze(
model.predict(np.expand_dims(lr, axis=0)))
if i == 1:
tmp = tmp * 255
tmp[tmp < 0] = 0
tmp[tmp > 255] = 255
sr = tmp.astype(np.uint8)
sr_tiles.append(sr)
elif i < 6:
sr_tiles.append(
Utils.denormalize(
np.squeeze(model.predict(
np.expand_dims(rescale_imgs_to_neg1_1(lr),
axis=0)),
axis=0)))
else:
sr_tiles.append(
cv2.resize(lr, (0, 0),
fx=4,
fy=4,
interpolation=cv2.INTER_NEAREST))
for lr in lr_tiles_overlap:
# fifth step: calculate the sr tiles
if i < 2:
if i == 1:
lr = lr.astype(np.float64)
lr = lr / 255
tmp = np.squeeze(
model.predict(np.expand_dims(lr, axis=0)))
if i == 1:
tmp = tmp * 255
tmp[tmp < 0] = 0
tmp[tmp > 255] = 255
sr = tmp.astype(np.uint8)
sr_tiles_overlap.append(sr)
elif i < 6:
sr_tiles_overlap.append(
Utils.denormalize(
np.squeeze(model.predict(
np.expand_dims(rescale_imgs_to_neg1_1(lr),
axis=0)),
axis=0)))
else:
sr_tiles_overlap.append(
cv2.resize(lr, (0, 0),
fx=4,
fy=4,
interpolation=cv2.INTER_NEAREST))
# sixth step: stitch the image
sr_simple = ImageStitching.stitch_images(
sr_tiles, test_pair[1].shape[1], test_pair[1].shape[0],
sr_tiles[0].shape[1], sr_tiles[0].shape[0],
test_pair[1].shape[1] // sr_tiles[0].shape[1],
test_pair[1].shape[0] // sr_tiles[0].shape[0])
sr_advanced = ImageStitching.stitching(
sr_tiles_overlap,
LR=None,
image_size=(test_pair[1].shape[0], test_pair[1].shape[1]),
adjustRGB=False,
overlap=True)
# seventh step: append the mean metric for this image
mse.append(metrics.MSE(test_pair[1], sr_simple))
psnr.append(metrics.PSNR(test_pair[1], sr_simple))
ssim.append(metrics.SSIM(test_pair[1], sr_simple))
mssim.append(metrics.MSSIM(test_pair[1], sr_simple))
o_mse.append(metrics.MSE(test_pair[1], sr_advanced))
o_psnr.append(metrics.PSNR(test_pair[1], sr_advanced))
o_ssim.append(metrics.SSIM(test_pair[1], sr_advanced))
o_mssim.append(metrics.MSSIM(test_pair[1], sr_advanced))
# ninth step: append the mean metric for this model
stitch_performance[model_names[i]] = [
(np.mean(mse), np.mean(psnr), np.mean(ssim), np.mean(mssim)),
(np.mean(o_mse), np.mean(o_psnr), np.mean(o_ssim),
np.mean(o_mssim))
]
# final output
print("Performance on stitched images:")
f = open("stitch_performance.txt", "w")
for key in stitch_performance:
print(
"simple stitch: " + key + ": MSE = " +
str(stitch_performance[key][0][0]) + ", PSNR = " +
str(stitch_performance[key][0][1]) + ", SSIM = " +
str(stitch_performance[key][0][2]),
", MSSIM = " + str(stitch_performance[key][0][3]))
print(
"advanced stitch: " + key + ": MSE = " +
str(stitch_performance[key][1][0]) + ", PSNR = " +
str(stitch_performance[key][1][1]) + ", SSIM = " +
str(stitch_performance[key][1][2]),
", MSSIM = " + str(stitch_performance[key][1][3]))
f.write(key + " " + str(stitch_performance[key][0][0]) + " " +
str(stitch_performance[key][0][1]) + " " +
str(stitch_performance[key][0][2]) + " " +
str(stitch_performance[key][0][3]) + "\n")
f.write(key + " " + str(stitch_performance[key][1][0]) + " " +
str(stitch_performance[key][1][1]) + " " +
str(stitch_performance[key][1][2]) + " " +
str(stitch_performance[key][1][3]) + "\n")
f.close()
def perform_analysis():
"""
:return:
"""
parser = parse_arg()
args = parser.parse_args()
df = pd.read_csv(args.input)
# df = df.apply(lambda x: preProcessing.scaling_y_data(x) if x.name == 'OC' else x) # scaling OC data
(cal_df, tst_df) = separating_data_set(df)
(X_train, y_train) = splitting_dataset(cal_df)
(X_test, y_test) = splitting_dataset(tst_df)
# Scale the features
X_train = preProcessing.scaler_min_max_x_data(X_train)
X_test = preProcessing.scaler_min_max_x_data(X_test)
y_train = preProcessing.scaler_min_max_y_data(y_train)
y_test = preProcessing.scaler_min_max_y_data(y_test)
print(X_train)
print(y_train)
print(X_test)
print(y_test)
print(X_train.shape)
print(y_train.shape)
if args.hiddenLayers == 5:
model = build_fnn_5l(X_train)
elif args.hiddenLayers == 4:
model = build_fnn_4l(X_train)
elif args.hiddenLayers == 3:
model = build_fnn_3l(X_train)
elif args.hiddenLayers == 2:
model = build_fnn_2l(X_train)
else:
model = build_fnn_1l(X_train)
model.summary()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
random.seed(1)
model.fit(X_train, y_train, batch_size=args.batchSize, epochs=args.epochSize, validation_data=(X_test, y_test),
callbacks=[early_stop])
fnn_losses = pd.DataFrame(model.history.history)
create_losses_plot(fnn_losses)
trained_model_save(model, "trained_model.h5")
predictions = model.predict(X_test)
print("MSE:", metrics.MSE(y_test, predictions))
print("RMSE:", metrics.RMSE(y_test, predictions))
print("R-square:", metrics.R2(y_test, predictions))
print("RPD:", metrics.RPD(y_test, predictions))
create_prediction_plot(y_test, predictions)
if not os.path.exists(save_dirA) or not os.path.exists(save_dirB):
os.makedirs(save_dirA)
os.makedirs(save_dirB)
### Build model
netG_A2C = eval(checkA[0].replace('@G2LAB', ''))(1, 1, int(checkA[2][1])).to(opt.device)
netG_C2B = eval(checkB[0].replace('@G2LAB', ''))(1, 2).to(opt.device)
# load check point
netG_A2C.load_state_dict(torch.load(os.path.join(Check_DIR, os.path.basename(args.netGA))))
netG_C2B.load_state_dict(torch.load(os.path.join(Check_DIR, os.path.basename(args.netGB))))
netG_A2C.eval()
netG_C2B.eval()
print("Starting test Loop...")
# setup data loader
data_loader = DataLoader(testset, opt.batch_size, num_workers=opt.num_works,
shuffle=False, pin_memory=True, )
evaluators = [metrics.MSE(), metrics.PSNR(), metrics.AE(), metrics.SSIM()]
performs = [[] for i in range(len(evaluators))]
for idx, sample in enumerate(data_loader):
realA = sample['src'].to(opt.device)
# realA -= 0.5
realB = sample['tar'].to(opt.device)
# realB -= 0.5
# Y = 0.2125 R + 0.7154 G + 0.0721 B [RGB2Gray, 3=>1 ch]
realBC = realB[:,:1,:,:]
sf = int(checkA[2][1])
realBA = nn.functional.interpolate(realBC, scale_factor=1. / sf, mode='bilinear')
realBA = nn.functional.interpolate(realBA, scale_factor=sf, mode='bilinear')
# realAA = nn.functional.interpolate(realA, scale_factor=1. / sf)
realAA = realA
fake_AC = netG_A2C(realAA)
fake_AB = netG_C2B(fake_AC)
def validate(self, test_as_well=False):
"""Validates all models for all polynomials and all parameters, and stores data in validation_errors.
Creates and populates pandas.DataFrame validation_errors with MSE from bootstrap and kfold resampling techniques,
as well as model bias and variance from bootstrap, for all combinations of hyperparameters.
Parameters:
-----------
test_as_well:
bool
If True, test all models on test set as well, and store results
"""
# Make dataframe for validation data
lambda_list = np.unique(np.concatenate(self.para_iters)) if len(self.para_iters) > 1 else self.para_iters[0]
validation_index = pd.MultiIndex.from_product([
['Boot MSE', 'Boot Bias', 'Boot Var', 'Kfold MSE', 'Test MSE'],
[model.name for model in self.models],
[_lambda for _lambda in lambda_list]],
names = ['Metric', 'Model', 'Lambda']
)
self.validation_errors = pd.DataFrame(dtype=float, index=validation_index, columns=self.poly_iter)
self.validation_errors.sort_index(inplace=True)
for i, p in enumerate(self.poly_iter):
if self.verbose >= 1:
print(f"{(i+1):2d}/{len(poly_iter)}: Polynomial of degree {p}")
X_train_validate = poly_design_matrix(p, self.data['x_train_validate'])
X_train, X_validate = np.split(X_train_validate, [self.data['x_train'].shape[0]])
for j, (model, para_iter) in enumerate(zip(self.models, self.para_iters)):
if self.verbose >= 1:
print(f" {(j+1):2d}/{len(self.models)}: Model: {model.name}")
for k, _lambda in enumerate(para_iter):
if self.verbose >= 2:
print(f" {(k+1):2d}/{len(para_iter)}: Lambda = {_lambda}")
model.set_lambda(_lambda)
boot_mse, boot_bias, boot_var = resampling.bootstrap(model, X_train, X_validate, self.data['y_train'], self.data['y_validate'], R=50)
kfold_mse = resampling.kfold(model, X_train_validate, self.data['y_train_validate'], n_folds=10)
self.validation_errors.loc['Boot MSE', model.name, _lambda][p] = boot_mse
self.validation_errors.loc['Boot Bias', model.name, _lambda][p] = boot_bias
self.validation_errors.loc['Boot Var', model.name, _lambda][p] = boot_var
self.validation_errors.loc['Kfold MSE', model.name, _lambda][p] = kfold_mse
if test_as_well:
X_train_c = np.copy(X_train)
model.scaler.fit(X_train_c[:,1:])
X_train_c[:,1:] = model.scaler.transform(X_train_c[:,1:])
model.set_lambda(_lambda)
model.fit(X_train_c, self.data['y_train'])
X_test = poly_design_matrix(p, self.data['x_test'])
X_test[:,1:] = model.scaler.transform(X_test[:,1:])
y_pred = model.predict(X_test)
test_mse = metrics.MSE(data['y_test'], y_pred)
self.validation_errors.loc['Test MSE', model.name, _lambda][p] = test_mse
#if self.verbose >= 2:
#print(f" error: {(boot_mse+kfold_mse)/2}")
self.validation_errors.dropna(thresh=2, inplace=True)
def regression_report(y_true, y_pred):
print('--------------------------------')
print('MSE -', metrics.MSE(y_true, y_pred))
print('RMSE -', metrics.RMSE(y_true, y_pred))
print('MAN -', metrics.MAN(y_true, y_pred))
print('--------------------------------')
def terrain_reggression(data, epochs=5000, epochs_without_progress=500, mini_batch_size=20):
"""Plots performance of different neural models on regression problem for real terrain"""
polynomials = 8
X_train = linear_models.poly_design_matrix(polynomials, data['x_train'])
X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate'])
X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])
total_steps = epochs * len(data['x_train'])//mini_batch_size
l_rate = learning_rate.Learning_rate(base=3e-2, decay=1/20000).ramp_up(10).compile(total_steps)
sigmoid = {'name': 'Sigmoid',
'layers': [{'height': 2},
{'height': 2},
{'height': 2},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
leaky = {'name': 'Leaky ReLu',
'layers': [{'height': 2},
{'height': 8, 'activations': activations.leaky_relus},
{'height': 8, 'activations': activations.leaky_relus},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
relu = {'name': 'ReLu',
'layers': [{'height': 2},
{'height': 8, 'activations': activations.relus},
{'height': 8, 'activations': activations.relus},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
neural_models = []
for model in [sigmoid, leaky, relu]:
neural_models.append(neural_model.Network(**model))
ridge_model = [linear_models.RegularisedLinearRegression(
name = 'Ridge',
beta_func = linear_models.beta_ridge,
momentum = 0.6,
_lambda = 0.001,
x_shape = X_train.shape[1],
learning_rate = 0.01
), X_train, X_validate, data['y_train'], data['y_validate']]
subplots = [([*neural_models, ridge_model], {'mini_batch_size': mini_batch_size})]
errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(data['x_train'], data['x_validate'], data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=epochs_without_progress)
title = ['Best models of each type, regression', 'regression_vs', subtitle]
sgd.plot_sgd_errors(errors, title, metrics_string)
print("Model performances on test data")
neural_subplots = []
for model in neural_models:
y_pred = model.predict(data['x_test'])
mse = metrics.MSE(data['y_test'], y_pred)
r2 = metrics.R_2(data['y_test'], y_pred)
neural_subplots.append((model.name, data['x_test'], y_pred[:,0]))
print(f"{model.name}: {mse} {r2}")
X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])
ridge_pred = ridge_model[0].predict(X_test)
mse = metrics.MSE(data['y_test'], ridge_pred)
r2 = metrics.R_2(data['y_test'], ridge_pred)
print(f"{ridge_model[0].name}: {mse} {r2}")
plotting.side_by_side(
*neural_subplots,
['Ridge', data['x_test'], ridge_pred[:,0]],
['Ground truth', data['x_test'], data['y_test'][:,0]],
title = ["Terrain predictions", "terrain_pred"],
projection = '3d',
plotter = plotting.trisurface_plotter,
view_angles = [30, 230])
X_test = tune_and_evaluate.poly_design_matrix(p, data['x_test'])
model = linear_models.LinearRegression("OLS")
model.scaler.fit(X_train[:, 1:])
X_train[:, 1:] = model.scaler.transform(X_train[:, 1:])
X_test[:, 1:] = model.scaler.transform(X_test[:, 1:])
model.fit(X_train, data['y_train'])
y_tilde = model.predict(X_train)
y_pred = model.predict(X_test)
conf_intervals, conf_deviation = model.conf_interval_beta(
data['y_train'], y_tilde, X_train)
errors['training'][p - 1] = np.array([
metrics.MSE(data['y_train'], y_tilde),
metrics.R_2(data['y_train'], y_tilde)
])
errors['testing'][p - 1] = np.array([
metrics.MSE(data['y_test'], y_pred),
metrics.R_2(data['y_test'], y_pred)
])
# Training vs testing, demonstrating overfitting
fig = plt.figure(figsize=(7, 7))
plt.plot(poly_iter, errors['training'][:, 0], label="Training")
plt.plot(poly_iter, errors['testing'][:, 0], label="Testing")
plt.legend()
plt.xlabel("Complexity (maximum degree)")
plt.ylabel("Mean squared error")
plt.savefig("../plots/overfitting_franke.png")
import matplotlib.pyplot as plt
import numpy as np
import metrics #Our custom library
# data generator part
x = np.arange(1000)
delta = np.random.uniform(-10,10, size=(len(x),)) # добавим случайный шум ,посмотрите, что лежит внутри
y_test = 0.4 * x +3 + delta # тестируемая выборка (полученная из эксперимента)
y_valid = 0.4*x + 3 # валидная выборка (правильный ответ без шума)
plt.plot(x,y_test, label = 'Experimental Data')
plt.plot(x, y_valid, label = 'Valid Data')
plt.title('Example of data sample')
plt.legend()
plt.show()
print('Your MSE is : ',metrics.MSE(y_test, y_valid))
#print('Your MAE is : ',metrics.MAE(y_test, y_valid)) #Uncomment this part after compliting metrics.py
#print('Your RMSE is : ',metrics.RMSE(y_test, y_valid))
#print('Your ME is : ',metrics.ME(y_test, y_valid))
#print('Your SD is : ',metrics.MSE(y_test, y_valid))
