for ds in datasets:
dataloader = DataLoader(dataset = ds,
batch_size = BATCH_SIZE,
shuffle = True,
num_workers = 0)
dataloader_list.append(dataloader)
data_num = dataset.data_num
sample_data_num = int(data_num/CV_NUM)
params = (QUERY_DIM,KEY_DIM,FEATURE_DIM)
## Attention Net
if NET == ATTENTION:
net = Attention_Net(dataset, params, activation = ACT)
## Linear Net
elif NET == LINEAR:
net = Linear_Net(dataset, FEATURE_DIM)
## Optimizer
if OPTIMIZER == SGD:
optimizer = torch.optim.SGD(net.parameters(), lr = LEARNING_RATE, momentum = MOMENTUM)
elif OPTIMIZER == ADAM:
optimizer = torch.optim.Adam(net.parameters(), lr = LEARNING_RATE, betas = BETAS)
## Loss
loss_function = torch.nn.MSELoss()
def grad_search(dataset, data_loader_list, grad_search_path, params=(1, 1, 1)):
QUERY_DIM = params[0]
KEY_DIM = params[1]
FEATURE_DIM = params[2]
############## Data Preparation ###################
model_path = "/home/li/torch/model/attention_net_Q_" + str(
QUERY_DIM) + "_K_" + str(KEY_DIM) + "_F_" + str(
FEATURE_DIM) + "_CV.model"
this_search_path = grad_search_path + "Q_" + str(QUERY_DIM) + "_K_" + str(
KEY_DIM) + "_F_" + str(FEATURE_DIM) + "/"
if not os.path.exists(this_search_path):
os.mkdir(this_search_path)
data_file_path = "/home/li/datasets/lifelog/data/Group1_li_mofei_no_20_20190520.csv"
user = "******"
train_log_file_path = this_search_path + "train_log.txt"
name_list = dataset.item_list
input_dim = len(name_list)
data_num = dataset.data_num
sample_data_num = int(data_num / CV_NUM)
## Attention Net
attention_net = Attention_Net(dataset, params, activation=ACT)
## Optimizer
if OPT == SGD:
optimizer = torch.optim.SGD(attention_net.parameters(),
lr=LEARNING_RATE,
momentum=MOMENTUM)
elif OPT == ADAM:
optimizer = torch.optim.Adam(attention_net.parameters(),
lr=LEARNING_RATE,
betas=BETAS)
## Train Log
with open(train_log_file_path, 'w') as train_f:
train_f.write("Optimizer: " + OPT + "\r\n")
train_f.write("Learning Rate: " + str(LEARNING_RATE) + "\r\n")
if OPT == SGD:
train_f.write("Momentum: " + str(MOMENTUM) + "\r\n")
elif OPT == ADAM:
train_f.write("Betas: " + str(BETAS) + "\r\n")
train_f.write("Regularization: " + str(REG) + "\r\n")
train_f.write("Loss: " + str(LOSS) + "\r\n")
train_f.write("Parameters: \r\n")
for name, param in attention_net.named_parameters():
if param.requires_grad:
train_f.write(str(name) + "\r\n")
train_log_file = open(train_log_file_path, 'a')
## Loss
if LOSS == MSE:
loss_function = torch.nn.MSELoss()
#### Print Parameters
#for name,param in attention_net.named_parameters():
# if param.requires_grad:
# print(name)
#print(param)
###################### Training ############### Cross Validation
#attention_net.train()
#print(dataloader)
train_loss_list = []
test_loss_list = []
class_1_distance_list = []
class_2_distance_list = []
class_1_star_distance_list = []
class_2_star_distance_list = []
inter_distance_list = []
coeff_1_list = []
coeff_2_list = []
coeff_3_list = []
for epoch in range(EPOCH):
train_loss_each_epoch_list = []
test_loss_each_epoch_list = []
for i in range(CV_NUM):
test_dataloader = dataloader_list[i]
train_dataloader_list = dataloader_list[:i] + dataloader_list[i +
1:]
attention_net.train()
#print(len(train_dataloader_list))
###### Train
train_loss_each_sample_list = []
for dataloader in train_dataloader_list:
train_loss_each = 0
for im, label in dataloader:
l0_regularization = torch.tensor(0).float()
l1_regularization = torch.tensor(0).float()
l2_regularization = torch.tensor(0).float()
out, dis = attention_net.forward(im)
mse_loss = loss_function(out, label)
## Regularization
for param in attention_net.parameters():
l1_regularization += WEIGHT_DECAY * torch.norm(
param, 1)
l2_regularization += WEIGHT_DECAY * torch.norm(
param, 2)
if REG == L0:
loss = mse_loss + l0_regularization
elif REG == L1:
loss = mse_loss + l1_regularization
elif REG == L2:
loss = mse_loss + l2_regularization
train_loss_each += mse_loss.item() / sample_data_num
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss_each_sample_list.append(train_loss_each)
#print(len(train_loss_each_sample_list))
#print(len(train_loss_each_sample_list))
train_loss_each_epoch_list.append(
np.mean(train_loss_each_sample_list))
############ Test
test_loss_each = 0
attention_net.eval()
for im, label in test_dataloader:
out, dis = attention_net.forward(im)
mse_loss = loss_function(out, label)
test_loss_each += mse_loss.item() / sample_data_num
test_loss_each_epoch_list.append(test_loss_each)
train_loss = np.mean(train_loss_each_epoch_list)
test_loss = np.mean(test_loss_each_epoch_list)
train_loss_list.append(train_loss)
test_loss_list.append(test_loss)
######################## Evaluation
info1 = "Epoch: " + str(epoch) + " , Train Loss: " + str(train_loss)
info2 = "Epoch: " + str(epoch) + " , Test Loss: " + str(test_loss)
class_1_distance, class_2_distance, class_1_star_distance, class_2_star_distance, inter_distance = evaluate_model_inner_inter_distance(
attention_net, sample_number=EVA_SAMPLE_NUMBER, order=ORDER)
class_1_distance_list.append(class_1_distance)
class_2_distance_list.append(class_2_distance)
class_1_star_distance_list.append(class_1_star_distance)
class_2_star_distance_list.append(class_2_star_distance)
inter_distance_list.append(inter_distance)
coeff_1 = float(class_1_star_distance / class_1_distance)
coeff_2 = float(class_2_star_distance / class_2_distance)
coeff_3 = float((class_1_star_distance + class_2_star_distance) /
(2 * inter_distance))
coeff_1_list.append(coeff_1)
coeff_2_list.append(coeff_2)
coeff_3_list.append(coeff_3)
info3 = "Epoch: " + str(epoch) + " , D11: " + str(
class_1_distance) + " , D22: " + str(class_2_distance)
info4 = "Epoch: " + str(epoch) + " , D11*: " + str(
class_1_star_distance) + ", D22*: " + str(
class_2_star_distance) + ", D12*: " + str(inter_distance)
info5 = "Epoch: " + str(epoch) + " , Coefficient 1: " + str(
coeff_1) + " , Coefficient 2: " + str(
coeff_2) + " , Coefficient 3: " + str(coeff_3)
train_log_file.write(info1 + "\r\n")
train_log_file.write(info2 + "\r\n")
train_log_file.write(info3 + "\r\n")
train_log_file.write(info4 + "\r\n")
train_log_file.write(info5 + "\r\n")
#torch.save(attention_net, model_path)
##################################### Get Output
#attention_net.eval()
#final_input = torch.from_numpy(np.ones(input_dim)).float().unsqueeze(0)
#final_output = attention_net.forward(final_input)
#final_matrix = attention_net.get_output_matrix(final_input, final_output,pandas = True)
#csv_output_file = "/home/li/torch/result/test_result_" + str(user) + ".csv"
#final_matrix.to_csv(csv_output_file)
#print(loss_list)
############################### Get Training Curve
extra = "_Optim_SGD_WD_L1_lr_05_epoach_50_"
figure = "train_curve"
plt_file = this_search_path + "Q_" + str(QUERY_DIM) + "_K_" + str(
KEY_DIM) + "_F_" + str(FEATURE_DIM) + str(extra) + str(figure) + ".png"
plt.plot(range(len(train_loss_list)), train_loss_list, label="train loss")
plt.plot(range(len(test_loss_list)), test_loss_list, label="test loss")
plt.legend(loc='upper right')
#plt.ylim(bottom = 0, top = 0.02)
plt.savefig(plt_file)
#plt.show()
plt.close('all')
############################ Get Distance Curve
figure = "distance_curve"
plt_file = this_search_path + "Q_" + str(QUERY_DIM) + "_K_" + str(
KEY_DIM) + "_F_" + str(FEATURE_DIM) + str(extra) + str(figure) + ".png"
plt.plot(range(len(class_1_distance_list)),
class_1_distance_list,
label="D11")
plt.plot(range(len(class_2_distance_list)),
class_2_distance_list,
label="D22")
plt.plot(range(len(class_1_star_distance_list)),
class_1_star_distance_list,
label="D11*")
plt.plot(range(len(class_2_star_distance_list)),
class_2_star_distance_list,
label="D22*")
plt.plot(range(len(inter_distance_list)),
inter_distance_list,
label="D12*")
plt.legend(loc='upper right')
plt.savefig(plt_file)
#plt.show()
plt.close('all')
############################ Get Coeff Curve
figure = "coefficient_curve"
plt_file = this_search_path + "Q_" + str(QUERY_DIM) + "_K_" + str(
KEY_DIM) + "_F_" + str(FEATURE_DIM) + str(extra) + str(figure) + ".png"
plt.plot(range(len(coeff_1_list)), coeff_1_list, label="C1")
plt.plot(range(len(coeff_2_list)), coeff_2_list, label="C2")
plt.plot(range(len(coeff_3_list)), coeff_3_list, label="C3")
plt.legend(loc='upper right')
plt.savefig(plt_file)
#plt.show()
plt.close('all')
############################ Get Coeff Scatter Plot Versus Loss
figure = "coefficient_scatter_plot"
plt_file = this_search_path + "Q_" + str(QUERY_DIM) + "_K_" + str(
KEY_DIM) + "_F_" + str(FEATURE_DIM) + str(extra) + str(figure) + ".png"
plt.scatter(train_loss_list,
coeff_1_list,
c="blue",
marker="o",
label="C1")
plt.scatter(train_loss_list,
coeff_2_list,
c="green",
marker="X",
label="C2")
plt.scatter(train_loss_list, coeff_3_list, c="red", marker="v", label="C3")
plt.legend(loc='upper right')
plt.savefig(plt_file)
#plt.show()
plt.close('all')
return_tuple = (train_loss_list[-1], test_loss_list[-1],
class_1_distance_list[-1], class_2_distance_list[-1],
class_1_star_distance_list[-1],
class_2_star_distance_list[-1], inter_distance_list[-1],
coeff_1_list[-1], coeff_2_list[-1], coeff_3_list[-1])
return return_tuple
def run_cross_validation(input_csv,output_csv,weight_csv,username,K_FOLDER,BATCH_NORM,MOMENTUM,ARTIFICIAL,DATE,EPOCH,KEY_DIM,FEATURE_DIM,QUERY_DIM,REG,LEARNING_RATE,WEIGHT_DECAY,LOSS,ACT,BATCH_SIZE,OPTIMIZER,NET,w_f,w_f_type,VALIDATE_NUMBER,WD,extra_msg,ATTENTION_REG, ATTENTION_REG_WEIGHT):
############### Data Preparation ##############
extra = "CV_" + str(extra_msg) + str(DATE) + "_Epoch_"+ str(EPOCH) + "_Net_" + str(NET) + "_u_" + str(username) + "_Q_" + str(QUERY_DIM) + "_K_" + str(KEY_DIM) + "_F_" + str(FEATURE_DIM) + "_REG_" + str(REG) + "_ACT_" + str(ACT) + "_WD_" + str(WD) + "w_f" + str(w_f_type)
#model_path = "/home/li/food/model/" + str(extra) + ".model"
## Artificial
dataset = FoodDataset(input_csv,output_csv)
print(weight_csv)
weights_df = pd.read_csv(weight_csv)
model_path = "/home/li/food/model/" + str(DATE) + "/"
plot_path = "/home/li/food/plot/" + str(DATE) + "/"
if not os.path.exists(plot_path):
os.mkdir(plot_path)
plot_path = plot_path + "CV/"
train_log_path = plot_path + "train_log/"
train_log_file_path = train_log_path + str(extra) + ".txt"
if not os.path.exists(plot_path):
os.mkdir(plot_path)
if not os.path.exists(model_path):
os.mkdir(model_path)
if not os.path.exists(train_log_path):
os.mkdir(train_log_path)
data_num = dataset.data_num
valid_data_num = int(data_num/K_FOLDER)
train_data_num = int(data_num - valid_data_num)
sample_data_num = int(valid_data_num)
print(train_data_num)
print(valid_data_num)
splits_list = []
for i in range(K_FOLDER):
splits_list.append(sample_data_num)
splits_list = tuple(splits_list)
print(splits_list)
datasets = torch.utils.data.random_split(dataset,splits_list)
dataloader_list = []
dataloader_bs1_list = []
#### DEBUG
#for ds in datasets:
#print(ds.__len__())
for ds in datasets:
dataloader = DataLoader(dataset = ds,
batch_size = BATCH_SIZE,
shuffle = True,
num_workers = 0)
dataloader_list.append(dataloader)
dataloader_bs1 = DataLoader(dataset = ds,
batch_size = 1,
shuffle = True,
num_workers = 0)
dataloader_bs1_list.append(dataloader_bs1)
params = (QUERY_DIM,KEY_DIM,FEATURE_DIM)
net_list = []
## NET
if NET == ATTENTION:
for i in range(K_FOLDER):
if w_f == "FIXED":
net = Attention_Net(dataset, params, activation = ACT, w_f = "Fixed", w_f_type = w_f_type)
elif w_f == "TRAIN":
net = Attention_Net(dataset, params, activation = ACT, w_f = "Train")
net_list.append(net)
elif NET == LINEAR:
for i in range(K_FOLDER):
net = Linear_Net(dataset, params = KEY_DIM, activation = ACT)
net_list.append(net)
elif NET == LINEAR_NOHIDDEN:
for i in range(K_FOLDER):
net = Linear_NoHidden_Net(dataset)
net_list.append(net)
### TEST
for k in range(K_FOLDER):
model_save = model_path + str(extra) + "_CV_Model_" + str(k) + "initial.model"
torch.save(net_list[k].state_dict(), model_save)
print(model_save)
optimizer_list = []
## OPTIMIZER
if OPTIMIZER == SGD:
for i in range(K_FOLDER):
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, net_list[i].parameters()), lr = LEARNING_RATE, momentum = MOMENTUM)
optimizer_list.append(optimizer)
elif OPTIMIZER == ADAM:
for i in range(K_FOLDER):
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net_list[i].parameters()), lr = LEARNING_RATE, betas = BETAS)
optimizer_list.append(optimizer)
## LOSS
if LOSS == MSE:
loss_function = torch.nn.MSELoss()
elif LOSS == CEL:
loss_function = torch.nn.CrossEntropyLoss()
########## TRAINING
train_loss_list = []
train_loss_log_list = []
valid_loss_list = []
valid_loss_log_list = []
valid_accurate_rate_list = []
train_accurate_rate_list = []
train_average_accuracy_list = []
valid_average_accuracy_list = []
for k in range(K_FOLDER):
train_loss_list.append([])
train_loss_log_list.append([])
valid_loss_list.append([])
valid_loss_log_list.append([])
valid_accurate_rate_list.append([])
train_accurate_rate_list.append([])
for epoch in range(EPOCH):
#dist_list = []
#valid_dist_list = []
#test_output_list = []
train_average_accuracy = 0
valid_average_accuracy = 0
for k in range(K_FOLDER):
valid_dataloader = dataloader_list[k]
train_dataloader = dataloader_list[:k] + dataloader_list[k+1:]
valid_dataloader_bs1 = dataloader_bs1_list[k]
train_dataloader_bs1 = dataloader_bs1_list[:k] + dataloader_bs1_list[k+1:]
net = net_list[k]
optimizer = optimizer_list[k]
net.eval()
train_loss_list_each_epoch = []
valid_loss_list_each_epoch = []
accurate_number = 0
for im,label in valid_dataloader_bs1:
if NET == ATTENTION:
out,dist_origin = net.forward(im,batch_norm = BATCH_NORM)
accurate_number += match_output(out,label.float())
else:
out = net.forward(im,batch_norm = BATCH_NORM)
accurate_number += match_output(out,label.float())
org_loss = loss_function(out, torch.max(label,1)[1])
valid_loss_list_each_epoch.append(org_loss.item())
valid_accurate_rate = float(accurate_number)/ valid_data_num
valid_accurate_rate_list[k].append(valid_accurate_rate)
valid_average_accuracy += valid_accurate_rate
valid_loss = np.mean(valid_loss_list_each_epoch)
valid_loss_list[k].append(valid_loss)
valid_loss_log_list[k].append(math.log(valid_loss))
accurate_number = 0
#counter = 0
for dataloader in train_dataloader_bs1:
train_loss_each = 0
for im,label in dataloader:
#counter += 1
if NET == ATTENTION:
out,dist_origin = net.forward(im,batch_norm = BATCH_NORM)
accurate_number += match_output(out,label.float())
#print(accurate_number)
else:
out = net.forward(im,batch_norm = BATCH_NORM)
accurate_number += match_output(out,label.float())
org_loss = loss_function(out, torch.max(label,1)[1])
train_loss_list_each_epoch.append(org_loss.item())
train_accurate_rate = float(accurate_number)/ train_data_num
train_accurate_rate_list[k].append(train_accurate_rate)
train_average_accuracy += train_accurate_rate
train_loss = np.mean(train_loss_list_each_epoch)
train_loss_list[k].append(train_loss)
train_loss_log_list[k].append(math.log(train_loss))
net.train()
for dataloader in train_dataloader:
for im,label in dataloader:
l0_regularization = torch.tensor(0).float()
l1_regularization = torch.tensor(0).float()
l2_regularization = torch.tensor(0).float()
attention_regularization = torch.tensor(0).float()
if NET == ATTENTION:
out,dist_origin = net.forward(im,batch_norm = BATCH_NORM)
#for dist in dist_origin:
#dist_list.append(list(dist.detach().numpy()))
if ATTENTION_REG:
for dist in dist_origin:
attention_regularization += ATTENTION_REG_WEIGHT * torch.norm(dist,0.5)
#print(attention_regularization)
else:
out = net.forward(im,batch_norm = BATCH_NORM)
org_loss = loss_function(out, torch.max(label,1)[1])
## Regularization
for param in net.parameters():
l1_regularization += WEIGHT_DECAY * torch.norm(param,1)
l2_regularization += WEIGHT_DECAY * torch.norm(param,2)
if REG == L0:
loss = org_loss + l0_regularization
elif REG == L1:
loss = org_loss + l1_regularization
elif REG == L2:
loss = org_loss + l2_regularization
if ATTENTION_REG:
loss = loss + attention_regularization
optimizer.zero_grad()
loss.backward()
optimizer.step()
### Train LOG
info1 = "Epoch: " + str(epoch) + " , CV_Model: " + str(k) + " , Train Loss: " + str(train_loss)
info2 = "Epoch: " + str(epoch) + " , CV_Model: " + str(k) + " , valid Loss: " + str(valid_loss)
info3 = "Epoch: " + str(epoch) + " , CV_Model: " + str(k) + " , Train Accuracy: " + str(train_accurate_rate) + " , Test Accuracy: " + str(valid_accurate_rate)
#print(info3)
info5 = "Epoch: " + str(epoch) + " , Output: " + str(out)
#print(info5)
if epoch % 50 == 0 or epoch == (EPOCH - 1):
print(info3)
for para in net.parameters():
info6 = "Epoch: " + str(epoch) + " , Parameters: " + str(para)
#print(info6)
train_average_accuracy_list.append(train_average_accuracy/K_FOLDER)
valid_average_accuracy_list.append(valid_average_accuracy/K_FOLDER)
if epoch % 100 == 0:
for k in range(K_FOLDER):
model_save = model_path + str(extra) + "_CV_Model_" + str(k) + ".model"
torch.save(net_list[k].state_dict(), model_save)
#### Model Save
#print(model_path)
for k in range(K_FOLDER):
model_save = model_path + str(extra) + "_CV_Model_" + str(k) + ".model"
torch.save(net_list[k].state_dict(), model_save)
print(model_save)
#### PLOT
figure = "Learning_Curve"
for k in range(K_FOLDER):
plt.figure()
title = "F_" + str(FEATURE_DIM) + "_K_"+ str(KEY_DIM) + "_wf_" + str(w_f) + "_Model_" + str(k)
plt_file = plot_path + str(extra) + "_" + str(figure) + "_CV_Model_" + str(k) + ".png"
#plt.plot(range(len(train_loss_list)), train_loss_list, label = "train loss")
plt.plot(range(len(train_loss_log_list[k])), train_loss_log_list[k], label = "log train loss")
plt.plot(range(len(valid_loss_log_list[k])), valid_loss_log_list[k], label = "log valid loss")
plt.title(title)
plt.legend(loc = "upper right")
plt.show()
plt.savefig(plt_file)
plt.close('all')
figure = "Accuracy_Curve"
for k in range(K_FOLDER):
plt.figure()
title = "F_" + str(FEATURE_DIM) + "_K_"+ str(KEY_DIM) + "_wf_" + str(w_f) + "_Model_" + str(k)
plt_file = plot_path + str(extra) + "_" + str(figure) + "_CV_Model_" + str(k) + ".png"
plt.plot(range(len(valid_accurate_rate_list[k])), valid_accurate_rate_list[k], label = "Valid Accurate Rate")
plt.plot(range(len(train_accurate_rate_list[k])), train_accurate_rate_list[k], label = "Train Accurate Rate")
plt.title(title)
plt.legend(loc = "lower right")
plt.show()
plt.savefig(plt_file)
plt.close('all')
figure = "Accuracy_Average_Curve"
title = "F_" + str(FEATURE_DIM) + "_K_"+ str(KEY_DIM) + "_wf_" + str(w_f)
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
plt.figure()
#plt.plot(range(len(train_loss_list)), train_loss_list, label = "train loss")
plt.plot(range(len(valid_average_accuracy_list)), valid_average_accuracy_list, label = "Valid Accurate Average")
plt.plot(range(len(train_average_accuracy_list)), train_average_accuracy_list, label = "Train Accurate Average")
plt.title(title)
plt.legend(loc = "lower right")
plt.savefig(plt_file)
plt.show()
plt.close('all')
figure = "Accuracy_Scatter"
plt.figure()
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
x = ["Train Accuracy","Test Accuracy"]
for k in range(K_FOLDER):
y = []
y.append(train_accurate_rate_list[k][-1])
y.append(valid_accurate_rate_list[k][-1])
label = "Model_" + str(k)
plt.scatter(x,y,s = 600, c = "blue", alpha = float(1)/K_FOLDER * (k+1), linewidths = "2", marker = "o", label = label)
y = []
y.append(train_average_accuracy_list[-1])
y.append(valid_average_accuracy_list[-1])
plt.scatter(x,y,s = 600, c = "yellow", alpha = 1, linewidths = "2", marker = "*", label = "Average")
plt.title(title)
plt.legend(loc = "best")
plt.show()
plt.savefig(plt_file)
plt.close('all')
if NET != ATTENTION:
return
data_list = []
counter = 0
for data in dataloader_bs1_list[0]:
counter += 1
data_list.append(data)
if counter >= VALIDATE_NUMBER:
break
for k in range(K_FOLDER):
net = net_list[k]
net.eval()
valid_dist_list = []
#valid_dataloader = dataloader_bs1_list[k]
for data in data_list:
out,dist_origin = net.forward(data[0],batch_norm = BATCH_NORM)
for dist in dist_origin:
valid_dist_list.append(list(dist.detach().numpy()))
pca = PCA(n_components = 2)
pca.fit(valid_dist_list)
valid_feature = pca.transform(valid_dist_list)
print("Model " + str(k))
print(pca.explained_variance_ratio_)
if len(pca.explained_variance_ratio_) >= 2:
figure = "PCA_Artificial_Train_Model_" + str(k)
title = "F_" + str(FEATURE_DIM) + "_K_"+ str(KEY_DIM) + "_wf_" + str(w_f) + "_Model_" + str(k)
plt.figure()
print(len(valid_feature[:,0]))
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
plt.scatter(valid_feature[:,0], valid_feature[:,1])
plt.grid()
plt.xlim(-1,1)
plt.ylim(-1,1)
plt.title(title)
plt.savefig(plt_file)
plt.show()
plt.close('all')
if FEATURE_DIM == 32:
matrix_value_csv = plot_path + str(extra) + "_value_matrix_Model_" + str(k) + ".csv"
matrix_value = []
for n,p in net_list[k].named_parameters():
if n == "value_matrix":
for i in range(KEY_DIM):
p_i = list(p[i].detach().numpy())
sortp = list(np.argsort(p_i))
matrix_value.append(p_i)
sort_index = []
for j in range(len(p_i)):
sort_index.append(int(sortp.index(j)))
matrix_value.append(sort_index)
matrix_value_df = pd.DataFrame(np.array(matrix_value).transpose())
matrix_value_df.to_csv(matrix_value_csv)
return
def run_normal(input_csv,output_csv,weight_csv,username,ARTIFICIAL,BATCH_NORM,DATE,EPOCH,KEY_DIM,FEATURE_DIM,QUERY_DIM,REG,LEARNING_RATE,WEIGHT_DECAY,LOSS,ACT,BATCH_SIZE,OPTIMIZER,NET,w_f,w_f_type,WD,MOMENTUM,extra_msg):
############### Data Preparation ##############
extra = str(extra_msg) + str(DATE) + "_Epoch_"+ str(EPOCH) + "_Net_" + str(NET) + "_u_" + str(username) + "_Q_" + str(QUERY_DIM) + "_K_" + str(KEY_DIM) + "_F_" + str(FEATURE_DIM) + "_REG_" + str(REG) + "_ACT_" + str(ACT) + "WD" + str(WD) + "_wf_" + str(w_f_type)
model_path = "/home/li/food/model/" + str(extra) + ".model"
valid_input_csv = "/home/li/food/data/20190903_limofei_100_input_validation.csv"
valid_output_csv = "/home/li/food/data/20190903_limofei_100_output_validation.csv"
valid_dataset = FoodDataset(valid_input_csv, valid_output_csv)
valid_dataloader = DataLoader(dataset = valid_dataset,
batch_size = 1,
shuffle = False,
num_workers = 0)
valid_data_num = valid_dataset.data_num
dataset = FoodDataset(input_csv,output_csv)
plot_path = "/home/li/food/plot/" + str(DATE) + "/"
train_log_path = plot_path + "train_log/"
train_log_file_path = train_log_path + str(extra) + ".txt"
if not os.path.exists(plot_path):
os.mkdir(plot_path)
if not os.path.exists(train_log_path):
os.mkdir(train_log_path)
data_num = dataset.data_num
train_data_num = data_num
dataloader = DataLoader(dataset = dataset,
batch_size = BATCH_SIZE,
shuffle = False,
num_workers = 0)
train_dataloader = DataLoader(dataset = dataset,
batch_size = 1,
shuffle = False,
num_workers = 0)
params = (QUERY_DIM,KEY_DIM,FEATURE_DIM)
## NET
if NET == ATTENTION:
if w_f == "FIXED":
net = Attention_Net(dataset, params, activation = ACT, w_f = "Fixed", w_f_type = w_f_type)
elif w_f == "TRAIN":
net = Attention_Net(dataset, params, activation = ACT, w_f = "Train")
elif NET == LINEAR:
net = Linear_Net(dataset, params = FEATURE_DIM, activation = ACT)
#for i in filter(lambda p: p.requires_grad, net.parameters()):
# print(i)
#for i in filter(lambda p: not p.requires_grad, net.parameters()):
# print(i)
## OPTIMIZER
if OPTIMIZER == SGD:
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr = LEARNING_RATE, momentum = MOMENTUM)
#optimizer = torch.optim.SGD(net.parameters(), lr = LEARNING_RATE, momentum = MOMENTUM)
elif OPTIMIZER == ADAM:
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr = LEARNING_RATE, betas = BETAS)
## LOSS
if LOSS == MSE:
loss_function = torch.nn.MSELoss()
elif LOSS == CEL:
loss_function = torch.nn.CrossEntropyLoss()
########## TRAINING
train_loss_list = []
train_loss_log_list = []
train_accurate_rate_list = []
if not ARTIFICIAL:
test_loss_list = []
test_loss_log_list = []
test_accurate_rate_list = []
for epoch in range(EPOCH):
net.train()
dist_list = []
valid_dist_list = []
if not ARTIFICIAL:
test_output_list = []
test_loss_list_each_epoch = []
train_loss_list_each_epoch = []
if not ARTIFICIAL:
accurate_number = 0
for im,label in valid_dataloader:
if NET == ATTENTION:
out,dist_origin = net.forward(im,batch_norm = BATCH_NORM)
for dist in dist_origin:
valid_dist_list.append(list(dist.detach().numpy()))
output_array = list(out.detach().numpy())
test_output_list.append(output_array)
accurate_number += match_output(out,label.float())
elif NET == LINEAR:
out = net.forward(im,batch_norm = BATCH_NORM)
output_array = list(out.detach().numpy())
test_output_list.append(output_array)
accurate_number += match_output(out,label.float())
org_loss = loss_function(out, torch.max(label,1)[1])
test_loss_list_each_epoch.append(org_loss.item())
valid_accurate_rate = float(accurate_number) / valid_data_num
test_accurate_rate_list.append(valid_accurate_rate)
if not ARTIFICIAL:
test_loss = np.mean(test_loss_list_each_epoch)
test_loss_list.append(test_loss)
test_loss_log_list.append(math.log(test_loss))
accurate_number = 0
for im,label in train_dataloader:
if NET == ATTENTION:
out,dist_origin = net.forward(im,batch_norm = BATCH_NORM)
for dist in dist_origin:
dist_list.append(list(dist.detach().numpy()))
output_array = list(out.detach().numpy())
#test_output_list.append(output_array)
accurate_number += match_output(out,label.float())
elif NET == LINEAR:
out = net.forward(im,batch_norm = BATCH_NORM)
output_array = list(out.detach().numpy())
#test_output_list.append(output_array)
accurate_number += match_output(out,label.float())
#org_loss = loss_function(out, torch.max(label,1)[1])
#test_loss_list_each_epoch.append(org_loss.item())
train_accurate_rate = float(accurate_number)/ train_data_num
train_accurate_rate_list.append(train_accurate_rate)
train_loss = np.mean(train_loss_list_each_epoch)
train_loss_list.append(train_loss)
train_loss_log_list.append(math.log(train_loss))
for im,label in dataloader:
l0_regularization = torch.tensor(0).float()
l1_regularization = torch.tensor(0).float()
l2_regularization = torch.tensor(0).float()
if NET == ATTENTION:
out,dist_origin = net.forward(im,masked = MASK,batch_norm = BATCH_NORM)
elif NET == LINEAR:
out = net.forward(im,batch_norm = BATCH_NORM)
elif NET == LINEAR_NOHIDDEN:
out = net.forward(im,batch_norm = BATCH_NORM)
#print(out)
#print(label)
#print(out.shape)
#print(label.shape)
org_loss = loss_function(out, torch.max(label,1)[1])
#print(torch.max(label,1)[1])
#print(org_loss)
## Regularization
for param in filter(lambda p: p.requires_grad, net.parameters()):
l1_regularization += WEIGHT_DECAY * torch.norm(param,1)
l2_regularization += WEIGHT_DECAY * torch.norm(param,2)
if REG == L0:
loss = org_loss + l0_regularization
elif REG == L1:
loss = org_loss + l1_regularization
elif REG == L2:
loss = org_loss + l2_regularization
train_loss_list_each_epoch.append(org_loss.item())
optimizer.zero_grad()
#print(loss)
loss.backward()
optimizer.step()
info1 = "Epoch: " + str(epoch) + " , Train Loss: " + str(train_loss)
print(info1)
if not ARTIFICIAL:
info2 = "Epoch: " + str(epoch) + " , Test Loss: " + str(test_loss)
print(info2)
if NET == ATTENTION:
info3 = "Epoch: " + str(epoch) + " , Distribution: " + str(dist_origin)
print(info3)
info4 = "Epoch: " + str(epoch) + " , Train Accuracy: " + str(train_accurate_rate)
print(info4)
if not ARTIFICIAL:
info5 = "Epoch: " + str(epoch) + " , Test Accuracy: " + str(valid_accurate_rate)
print(info5)
for para in net.parameters():
info6 = "Epoch: " + str(epoch) + " , Parameters: " + str(para)
#print(info6)
print(model_path)
torch.save(net.state_dict(), model_path)
model = net
#### PLOT
figure = "Learning_Curve"
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
#plt.plot(range(len(train_loss_list)), train_loss_list, label = "train loss")
plt.plot(range(len(train_loss_log_list)), train_loss_log_list, label = "log train loss")
if not ARTIFICIAL:
plt.plot(range(len(test_loss_log_list)), test_loss_log_list, label = "log test loss")
plt.legend(loc = "upper right")
plt.savefig(plt_file)
plt.close('all')
figure = "Accurate_Rate_Curve"
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
#plt.plot(range(len(train_loss_list)), train_loss_list, label = "train loss")
if not ARTIFICIAL:
plt.plot(range(len(test_accurate_rate_list)), test_accurate_rate_list, label = "Valid Accurate Rate")
plt.plot(range(len(train_accurate_rate_list)), train_accurate_rate_list, label = "Train Accurate Rate")
plt.legend(loc = "lower right")
plt.savefig(plt_file)
plt.close('all')
##### TEST
if NET == ATTENTION:
pca = PCA(n_components = 'mle')
pca.fit(dist_list)
feature = pca.transform(dist_list)
print(pca.explained_variance_ratio_)
figure = "PCA_train"
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
plt.scatter(feature[:,0], feature[:,1])
plt.grid()
plt.xlim(-1,1)
plt.ylim(-1,1)
plt.savefig(plt_file)
plt.close('all')
pca_valid = PCA(n_components = 'mle')
pca_valid.fit(valid_dist_list)
valid_feature = pca_valid.transform(valid_dist_list)
print(pca_valid.explained_variance_ratio_)
figure = "PCA_valid"
print(len(valid_feature[:,0]))
plt_file = plot_path + str(extra) + "_" + str(figure) + ".png"
plt.scatter(valid_feature[:,0], valid_feature[:,1])
plt.grid()
plt.xlim(-1,1)
plt.ylim(-1,1)
plt.savefig(plt_file)
plt.close('all')
with open(train_log_file_path,"w") as log_f:
log_f.write(info1 + "\r\n")
log_f.write(info2 + "\r\n")
if NET == ATTENTION:
log_f.write(info3 + "\r\n")
log_f.write(info4 + "\r\n")
if NET == ATTENTION:
log_f.write("Variance Ratio Train:" + str(pca.explained_variance_ratio_) + "\r\n")
log_f.write("Variance Ratio Test:" + str(pca_valid.explained_variance_ratio_) + "\r\n")
return
dataloader_bs1 = DataLoader(dataset = ds,
batch_size = 1,
shuffle = True,
num_workers = 0)
dataloader_bs1_list.append(dataloader_bs1)
params = (QUERY_DIM,KEY_DIM,FEATURE_DIM)
net_list = []
## NET
if NET == ATTENTION:
for i in range(K_FOLDER):
net = Attention_Net(dataset, params, activation = ACT)
net_list.append(net)
elif NET == LINEAR:
for i in range(K_FOLDER):
net = Linear_Net(dataset, params = FEATURE_DIM, activation = ACT)
net_list.append(net)
optimizer_list = []
## OPTIMIZER
if OPTIMIZER == SGD:
for i in range(K_FOLDER):
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, net_list[i].parameters()), lr = LEARNING_RATE, momentum = MOMENTUM)
optimizer_list.append(optimizer)
elif OPTIMIZER == ADAM:
dataloader = DataLoader(dataset=dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0)
train_dataloader = DataLoader(dataset=dataset,
batch_size=1,
shuffle=False,
num_workers=0)
params = (QUERY_DIM, KEY_DIM, FEATURE_DIM)
## NET
if NET == ATTENTION:
net = Attention_Net(dataset, params, activation=ACT)
elif NET == LINEAR:
net = Linear_Net(dataset, params=FEATURE_DIM, activation=ACT)
for i in filter(lambda p: p.requires_grad, net.parameters()):
print(i)
for i in filter(lambda p: not p.requires_grad, net.parameters()):
print(i)
## OPTIMIZER
if OPTIMIZER == SGD:
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
net.parameters()),
lr=LEARNING_RATE,
momentum=MOMENTUM)
#optimizer = torch.optim.SGD(net.parameters(), lr = LEARNING_RATE, momentum = MOMENTUM)
username) + "_output_mds_figure.png"
csv_path = "/home/li/torch/evaluation/" + str(extra) + "_object_8_" + str(
username) + "_output_distance.csv"
bar_path = "/home/li/torch/evaluation/" + str(extra) + "_bar_graph_" + str(
username) + ".png"
item_name_path = "/home/li/torch/evaluation/" + str(
extra) + "_item_name_" + str(username) + ".txt"
coeff_path = "/home/li/torch/artificial_data/coefficient_log_30000_test_L2_WD_00001.txt"
dataset = GlobalModelDataset(input_csv, output_csv)
params = (9, 6, 5)
model = Attention_Net(dataset, params, activation=ACT)
model.load_state_dict(torch.load(model_path))
model.eval()
for name, param in model.named_parameters():
if param.requires_grad:
print(name, param.data)
item_list = model.item_list
class_1_list = item_list[:32]
class_2_list = item_list[32:]
print(item_list)
embedding = MDS(n_components=2, dissimilarity="precomputed")
for ds in datasets:
dataloader = DataLoader(dataset=ds,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0)
dataloader_list.append(dataloader)
data_num = dataset.data_num
sample_data_num = int(data_num / CV_NUM)
params = (QUERY_DIM, KEY_DIM, FEATURE_DIM)
## Attention Net
if NET == ATTENTION:
net = Attention_Net(dataset, params, activation=ACT)
## Linear Net
elif NET == LINEAR:
net = Linear_Net(dataset, FEATURE_DIM)
## Optimizer
if OPTIMIZER == SGD:
optimizer = torch.optim.SGD(net.parameters(),
lr=LEARNING_RATE,
momentum=MOMENTUM)
elif OPTIMIZER == ADAM:
optimizer = torch.optim.Adam(net.parameters(),
lr=LEARNING_RATE,
betas=BETAS)
## Loss
output_csv = "/home/li/food/artificial_data/Artificial_20191007_ITEM_NO_32_CLASS_NO_2_DATA_NO_100000_CHOICE_NO_4_CONDITION_NO32output.csv"
dataset = FoodDataset(input_csv, output_csv)
#print(dataset[:100])
plot_path = "/home/li/food/plot/" + str(DATE) + "/CV/"
model_path = "/home/li/food/model/"
if not os.path.exists(plot_path):
os.mkdir(plot_path)
params = (QUERY_DIM, KEY_DIM, FEATURE_DIM)
net_list = []
for i in range(K_FOLDER):
model_file = model_path + "Data_1000_Epoch_20191007_100_Net_attention_net_u_artificial_Q_9_K_6_F_5_REG_L2_ACT_sigmoid_WD_0005_MASK_True_CV_Model_" + str(
i) + ".model"
if NET == ATTENTION:
net = Attention_Net(dataset, params, activation=ACT)
net_list.append(net)
dataloader = DataLoader(dataset=dataset,
batch_size=1,
shuffle=True,
num_workers=0)
counter = 0
data_list = []
for i in dataloader:
counter += 1
data_list.append(i)
if counter >= VALIDATE_NUMBER:
break
#print(i[0])