N_selection = 50
improv_thresh = 1e-3
print("Step 1.")
weight_ga = GeneticAlgorithm(3, N_init_pop, mu=0.1)
weight_pop = weight_ga.get_population()
metric_array = np.empty(N_init_pop)
for i in range(len(weight_pop)):
#Scale input data
scaled_x_train = np.multiply(x_train, weight_pop[i])
#Scale verificaion data
scaled_x_verif = np.multiply(x_verif, weight_pop[i])
#Method 1
reg = KNNRegressor(scaled_x_train, y_train, 5)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
metric_array[i] = nbh_std
#Update fitness in GA object
weight_ga.set_fitness(metric_array)
weight_ga.selection(N_selection)
new_best_metric = 2.5
#while (best_metric - new_best_metric) > improv_thresh:
count = 0
print("About to start GA.")
while (count < 30):
count += 1
best_metric = new_best_metric
# Values of parameter k to iterate over.
K_VALS = [3, 5, 7, 9, 11, 13, 15]
starttime = time.time()
# Repeat each trial 10 times.
for i in range (0, 10):
x_train, x_test, y_train, y_test = train_test_split(X, y,\
test_size=0.2)
"""
Try non-optimized methods.
"""
# Vanilla KNN.
for k in K_VALS:
reg = KNNRegressor(x_train, y_train, k)
y_pred = reg.predict(x_test)
mse_iter = skmse(y_test, y_pred)
print("xx,knn,", k,",", mse_iter)
# Distance-weighted KNN.
for k in K_VALS:
reg = DwKNNRegressor(x_train, y_train, k)
y_pred = reg.predict(x_test)
mse_iter = skmse(y_test, y_pred)
print("xx,dknn,", k,",", mse_iter)
"""
PCA with KNN.
"""
pca = PCA(n_components = 6)
#data_obj = CosineData(0, np.pi/2 + 0.5, # [0, np.pi/2, np.pi], # [0.3, 0.4, 0.3], 0.0) data_obj.generate() X, y = data_obj.get_rn() #plt.scatter(X, y) test_data_x = np.linspace(0.01, np.pi / 2 + 1.4, 100) test_data_y = np.cos(test_data_x) plt.plot(test_data_x, test_data_y) #Method 1 reg = KNNRegressor(X, y, 5) test_data_pred = reg.predict(test_data_x) plt.plot(test_data_x, test_data_pred, label='KNN Regressor') #Method 2 reg = DKNNRegressor(X, y, 5) test_data_pred = reg.predict(test_data_x) plt.plot(test_data_x, test_data_pred, label='Distance KNN Regressor') """ #Method 3 reg = ExpKNNRegressor(X, y, 5) test_data_pred = reg.predict(test_data_x) plt.plot(test_data_x, test_data_pred, label='Exp KNN Regressor') #Method 4 reg = WeightedExpKNNRegressor(X, y, 5, 10, 100, 10)
def ga_run(x_train, y_train, x_test, y_test, x_verif, y_verif, k):
# Run GA to find best weights.
N_init_pop = 50
N_crossover = 50
N_selection = 20
improv_thresh = 1e-3
_, nFeats = np.shape(x_train)
weight_ga = GeneticAlgorithm(nFeats, N_init_pop, mu=0.1)
weight_pop = weight_ga.get_population()
metric_array = np.empty(N_init_pop)
# Create the initial population.
for i in range(len(weight_pop)):
# Scale input data
scaled_x_train = np.multiply(x_train, weight_pop[i])
# Scale verificaion data
scaled_x_verif = np.multiply(x_verif, weight_pop[i])
# Regressor.
reg = KNNRegressor(scaled_x_train, y_train, k)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
metric_array[i] = nbh_std
# Update fitness in GA object.
weight_ga.set_fitness(metric_array)
weight_ga.selection(N_selection)
new_best_metric = 2.5
# while (best_metric - new_best_metric) > improv_thresh:
count = 0
while (count < 20):
count += 1
best_metric = new_best_metric
# Crossover.
weight_ga.crossover(N_crossover)
# Get new population.
weight_pop = weight_ga.get_population()
metric_array = np.empty(N_crossover)
# Evaluate and set fitness.
for i in range(len(weight_pop)):
# Scale input data
scaled_x_train = np.multiply(x_train, weight_pop[i])
# Scale verificaion data
scaled_x_verif = np.multiply(x_verif, weight_pop[i])
# Regressor.
reg = KNNRegressor(scaled_x_train, y_train, k)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
metric_array[i] = nbh_std
# Update fitness in GA object
weight_ga.set_fitness(metric_array)
# get_best_sol
best_weights, new_best_metric = weight_ga.best_sol()
#print("Metric of this iteration are: ", new_best_metric)
weight_ga.selection(N_selection)
# print("Best weights = ", best_weights, "\tBest metric = ", new_best_metric)
# Test with scaling after GA
# Concatenate training and verification sets.
x_train = np.concatenate((x_train, x_verif), axis=0)
y_train = np.concatenate([y_train, y_verif])
# Print the results of KNN.
reg = KNNRegressor(np.multiply(x_train, best_weights), y_train, k)
y_pred = reg.predict(np.multiply(x_test, best_weights))
mse_iter = skmse(y_test, y_pred)
print("ga,knn,", k, ",", mse_iter)
# Print the results of KNN.
reg = DwKNNRegressor(np.multiply(x_train, best_weights), y_train, k)
y_pred = reg.predict(np.multiply(x_test, best_weights))
mse_iter = skmse(y_test, y_pred)
print("ga,dknn,", k, ",", mse_iter)
def lbest_pso_run(x_train, y_train, x_test, y_test, x_verif, y_verif, k):
#Run PSO to find best weights
N_init_pop = 50
_, nFeats = np.shape(x_train)
weight_pso = LBestPSO(nFeats, N_init_pop)
pos = weight_pso.get_positions()
pbest = weight_pso.get_pbest()
pbest_metric_array = np.empty(N_init_pop)
pos_metric_array = np.empty(N_init_pop)
#Set pbest metrics
for i in range(len(pbest)):
#Scale input data
scaled_x_train = np.multiply(x_train, pbest[i])
#Scale verificaion data
scaled_x_verif = np.multiply(x_verif, pbest[i])
#Method 1
reg = KNNRegressor(scaled_x_train, y_train, k)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
pbest_metric_array[i] = nbh_std
weight_pso.set_pbest_fitness(pbest_metric_array)
#Set pos metrics
for i in range(len(pbest)):
#Scale input data
scaled_x_train = np.multiply(x_train, pos[i])
#Scale verificaion data
scaled_x_verif = np.multiply(x_verif, pos[i])
#Method 1
reg = KNNRegressor(scaled_x_train, y_train, k)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
pos_metric_array[i] = nbh_std
weight_pso.set_p_fitness(pos_metric_array)
#Set initial gbest.
weight_pso.set_init_best(pos_metric_array)
count = 0
while (count < 50):
count += 1
weight_pso.optimize()
#get_population
weight_pop = weight_pso.get_positions()
metric_array = np.empty(N_init_pop)
#evaluate and set fitness
for i in range(len(weight_pop)):
#Scale input data
scaled_x_train = np.multiply(x_train, weight_pop[i])
#Scale verificaion data
scaled_x_verif = np.multiply(x_verif, weight_pop[i])
#Method 1
reg = KNNRegressor(scaled_x_train, y_train, k)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
metric_array[i] = nbh_std
weight_pso.set_p_fitness(metric_array)
weight_pso.set_best(metric_array)
#get_best_sol
best_metric = weight_pso.get_gbest_fit()
best_weights = weight_pso.get_gbest()
# Concatenate training and verification sets.
x_train = np.concatenate((x_train, x_verif), axis=0)
y_train = np.concatenate([y_train, y_verif])
# Print the results of KNN.
reg = KNNRegressor(np.multiply(x_train, best_weights), y_train, k)
y_pred = reg.predict(np.multiply(x_test, best_weights))
mse_iter = skmse(y_test, y_pred)
print("lbest-pso,knn,", k, ",", mse_iter)
# Print the results of KNN.
reg = DwKNNRegressor(np.multiply(x_train, best_weights), y_train, k)
y_pred = reg.predict(np.multiply(x_test, best_weights))
mse_iter = skmse(y_test, y_pred)
print("lbest-pso,dknn,", k, ",", mse_iter)
train_data = np.genfromtxt(train_filename, delimiter=",")
test_data = np.genfromtxt(test_filename, delimiter=",")
x_train = train_data[:, :-1]
y_train = train_data[:, -1]
x_test = test_data[:, :-1]
y_test = test_data[:, -1]
#Generate verification data
X_1 = np.random.multivariate_normal(mean_1, cov_1, 5000)
X_2 = np.random.multivariate_normal(mean_2, cov_2, 5000)
x_verif = np.concatenate((X_1, X_2), axis=0)
#Method 1
reg = KNNRegressor(x_train, y_train, 5)
neighbors = reg.find_all_neighbors(x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
y_pred = reg.predict(x_test)
print(train_filename)
print(test_filename)
print(nbh_std)
print(mean_squared_error(y_test, y_pred)**0.5)
metric_array.append(nbh_std)
mse_array.append(mean_squared_error(y_test, y_pred)**0.5)
del train_data
del test_data
plot_x = [i for i in range(0, len(metric_array))]
improv_thresh = 1e-3
print("Step 1.")
# weight_ga = GeneticAlgorithm(3, N_init_pop, mu = 0.1)
weight_ga = GeneticAlgorithm(2, N_init_pop, mu=0.1)
weight_pop = weight_ga.get_population()
metric_array = np.empty(N_init_pop)
for i in range(len(weight_pop)):
#Scale input data
scaled_x_train = np.multiply(x_train, weight_pop[i])
#Scale verificaion data
scaled_x_verif = np.multiply(x_verif, weight_pop[i])
#Method 1
reg = KNNRegressor(scaled_x_train, y_train, 5)
neighbors = reg.find_all_neighbors(scaled_x_verif)
nbh_std = reg.find_neighborhood_std(neighbors)
metric_array[i] = nbh_std
#Update fitness in GA object
weight_ga.set_fitness(metric_array)
weight_ga.selection(N_selection)
new_best_metric = 2.5
#while (best_metric - new_best_metric) > improv_thresh:
count = 0
print("About to start GA.")
while (count < 15):
count += 1
best_metric = new_best_metric