def evaluate(save_model): print() print("--------Evaluating DATE model---------") # create best model best_model = torch.load(model_path) best_model.eval() # get threshold y_prob, val_loss = best_model.module.eval_on_batch(valid_loader) best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy) # predict test y_prob, val_loss = best_model.module.eval_on_batch(test_loader) overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_threshold) best_score = f1s[0] os.system("rm %s"%model_path) if save_model: scroed_name = "./saved_models/%s_%.4f.pkl" % (model_name,overall_f1) torch.save(best_model,scroed_name) return overall_f1, auc, precisions, recalls, f1s, revenues
def evaluate(save_model, exp_id): print() print("--------Evaluating DATE model---------") # create best model best_model = torch.load(model_path) best_model.eval() # get threshold y_prob, val_loss = best_model.eval_on_batch(valid_loader) best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy) # predict test y_prob, val_loss = best_model.eval_on_batch(test_loader) overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_threshold) best_score = f1s[0] os.system("rm %s"%model_path) if save_model: scroed_name = f'./saved_models/DATE_{starting_date}_{exp_id}_{round(overall_f1, 4)}.pkl' torch.save(best_model,scroed_name) return overall_f1, auc, precisions, recalls, f1s, revenues
def train(args): # get configs epochs = args.epoch dim = args.dim lr = args.lr weight_decay = args.l2 head_num = args.head_num device = args.device act = args.act fusion = args.fusion beta = args.beta alpha = args.alpha use_self = args.use_self agg = args.agg model = DATE(leaf_num,importer_size,item_size,\ dim,head_num,\ fusion_type=fusion,act=act,device=device,\ use_self=use_self,agg_type=agg, ).to(device) model = nn.DataParallel(model,device_ids=[0,1]) # initialize parameters for p in model.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) # optimizer & loss optimizer = Ranger(model.parameters(), weight_decay=weight_decay,lr=lr) cls_loss_func = nn.BCELoss() reg_loss_func = nn.MSELoss() # save best model global_best_score = 0 model_state = None # early stop settings stop_rounds = 3 no_improvement = 0 current_score = None for epoch in range(epochs): for step, (batch_feature,batch_user,batch_item,batch_cls,batch_reg) in enumerate(train_loader): model.train() # prep to train model batch_feature,batch_user,batch_item,batch_cls,batch_reg = \ batch_feature.to(device), batch_user.to(device), batch_item.to(device),\ batch_cls.to(device), batch_reg.to(device) batch_cls,batch_reg = batch_cls.view(-1,1), batch_reg.view(-1,1) # model output classification_output, regression_output, hidden_vector = model(batch_feature,batch_user,batch_item) # FGSM attack adv_vector = fgsm_attack(model,cls_loss_func,hidden_vector,batch_cls,0.01) adv_output = model.module.pred_from_hidden(adv_vector) # calculate loss adv_loss_func = nn.BCELoss(weight=batch_cls) adv_loss = beta * adv_loss_func(adv_output,batch_cls) cls_loss = cls_loss_func(classification_output,batch_cls) revenue_loss = alpha * reg_loss_func(regression_output, batch_reg) loss = cls_loss + revenue_loss + adv_loss optimizer.zero_grad() loss.backward() optimizer.step() if (step+1) % 1000 ==0: print("CLS loss:%.4f, REG loss:%.4f, ADV loss:%.4f, Loss:%.4f"\ %(cls_loss.item(),revenue_loss.item(),adv_loss.item(),loss.item())) # evaluate model.eval() print("Validate at epoch %s"%(epoch+1)) y_prob, val_loss = model.module.eval_on_batch(valid_loader) y_pred_tensor = torch.tensor(y_prob).float().to(device) best_threshold, val_score, roc = torch_threshold(y_prob,xgb_validy) overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_validy,revenue_valid) select_best = np.mean(f1s) print("Over-all F1:%.4f, AUC:%.4f, F1-top:%.4f" % (overall_f1, auc, select_best) ) print("Evaluate at epoch %s"%(epoch+1)) y_prob, val_loss = model.module.eval_on_batch(test_loader) y_pred_tensor = torch.tensor(y_prob).float().to(device) overall_f1, auc, precisions, recalls, f1s, revenues = metrics(y_prob,xgb_testy,revenue_test,best_thresh=best_threshold) print("Over-all F1:%.4f, AUC:%.4f, F1-top:%.4f" %(overall_f1, auc, np.mean(f1s)) ) # save best model if select_best > global_best_score: global_best_score = select_best torch.save(model,model_path) # early stopping if current_score == None: current_score = select_best continue if select_best < current_score: current_score = select_best no_improvement += 1 if no_improvement >= stop_rounds: print("Early stopping...") break if select_best > current_score: no_improvement = 0 current_score = None