def evaluate(save_model):
print()
print("--------Evaluating DATE model---------")
# create best model
best_model = torch.load(model_path)
best_model.eval()
# get threshold
y_prob, val_loss = best_model.module.eval_on_batch(valid_loader)
best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy)
# predict test
y_prob, val_loss = best_model.module.eval_on_batch(test_loader)
overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_threshold)
best_score = f1s[0]
os.system("rm %s"%model_path)
if save_model:
scroed_name = "./saved_models/%s_%.4f.pkl" % (model_name,overall_f1)
torch.save(best_model,scroed_name)
return overall_f1, auc, precisions, recalls, f1s, revenues
def evaluate(save_model, exp_id):
print()
print("--------Evaluating DATE model---------")
# create best model
best_model = torch.load(model_path)
best_model.eval()
# get threshold
y_prob, val_loss = best_model.eval_on_batch(valid_loader)
best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy)
# predict test
y_prob, val_loss = best_model.eval_on_batch(test_loader)
overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_threshold)
best_score = f1s[0]
os.system("rm %s"%model_path)
if save_model:
scroed_name = f'./saved_models/DATE_{starting_date}_{exp_id}_{round(overall_f1, 4)}.pkl'
torch.save(best_model,scroed_name)
return overall_f1, auc, precisions, recalls, f1s, revenues
def train(args):
# get configs
epochs = args.epoch
dim = args.dim
lr = args.lr
weight_decay = args.l2
head_num = args.head_num
device = args.device
act = args.act
fusion = args.fusion
beta = args.beta
alpha = args.alpha
use_self = args.use_self
agg = args.agg
model = DATE(leaf_num,importer_size,item_size,\
dim,head_num,\
fusion_type=fusion,act=act,device=device,\
use_self=use_self,agg_type=agg,
).to(device)
model = nn.DataParallel(model,device_ids=[0,1])
# initialize parameters
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
# optimizer & loss
optimizer = Ranger(model.parameters(), weight_decay=weight_decay,lr=lr)
cls_loss_func = nn.BCELoss()
reg_loss_func = nn.MSELoss()
# save best model
global_best_score = 0
model_state = None
# early stop settings
stop_rounds = 3
no_improvement = 0
current_score = None
for epoch in range(epochs):
for step, (batch_feature,batch_user,batch_item,batch_cls,batch_reg) in enumerate(train_loader):
model.train() # prep to train model
batch_feature,batch_user,batch_item,batch_cls,batch_reg = \
batch_feature.to(device), batch_user.to(device), batch_item.to(device),\
batch_cls.to(device), batch_reg.to(device)
batch_cls,batch_reg = batch_cls.view(-1,1), batch_reg.view(-1,1)
# model output
classification_output, regression_output, hidden_vector = model(batch_feature,batch_user,batch_item)
# FGSM attack
adv_vector = fgsm_attack(model,cls_loss_func,hidden_vector,batch_cls,0.01)
adv_output = model.module.pred_from_hidden(adv_vector)
# calculate loss
adv_loss_func = nn.BCELoss(weight=batch_cls)
adv_loss = beta * adv_loss_func(adv_output,batch_cls)
cls_loss = cls_loss_func(classification_output,batch_cls)
revenue_loss = alpha * reg_loss_func(regression_output, batch_reg)
loss = cls_loss + revenue_loss + adv_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (step+1) % 1000 ==0:
print("CLS loss:%.4f, REG loss:%.4f, ADV loss:%.4f, Loss:%.4f"\
%(cls_loss.item(),revenue_loss.item(),adv_loss.item(),loss.item()))
# evaluate
model.eval()
print("Validate at epoch %s"%(epoch+1))
y_prob, val_loss = model.module.eval_on_batch(valid_loader)
y_pred_tensor = torch.tensor(y_prob).float().to(device)
best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy)
overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_validy,revenue_valid)
select_best = np.mean(f1s)
print("Over-all F1:%.4f, AUC:%.4f, F1-top:%.4f" % (overall_f1, auc, select_best) )
print("Evaluate at epoch %s"%(epoch+1))
y_prob, val_loss = model.module.eval_on_batch(test_loader)
y_pred_tensor = torch.tensor(y_prob).float().to(device)
overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_thresh=best_threshold)
print("Over-all F1:%.4f, AUC:%.4f, F1-top:%.4f" %(overall_f1, auc, np.mean(f1s)) )
# save best model
if select_best > global_best_score:
global_best_score = select_best
torch.save(model,model_path)
# early stopping
if current_score == None:
current_score = select_best
continue
if select_best < current_score:
current_score = select_best
no_improvement += 1
if no_improvement >= stop_rounds:
print("Early stopping...")
break
if select_best > current_score:
no_improvement = 0
current_score = None