def evaluate(model, criterion, loader, verbose=True): model.eval() pred = [] gold = [] for X_1, X_2, X_3, x1_len, x2_len, x3_len, y, ind, X_text in loader: if x1_len is None: pred_prob = model(X_1, X_2, X_3) else: pred_prob = model(X_1, X_2, X_3, x1_len, x2_len, x3_len) # pred_prob = model(X_1,X_2,X_3,x1_len,x2_len,x3_len) pred.append(pred_prob[0].detach().cpu().numpy()) gold.append(y.cpu().numpy()) pred = np.concatenate(pred) gold = np.concatenate(gold) accuracy, microPrecision, microRecall, microF1 = getMetrics( pred, gold, verbose) return microF1
if i == 0: txtfile = open(txt_file, 'a') txtfile.write("\n--------------------\n") txtfile.write("Classifier %s, Parameters: %f, %f, %f" % (classifier, parameter_list[0], parameter_list[1], parameter_list[2])) txtfile.close() y_pred_val = model.predict(X_val.reshape(X_val.shape[0], -1)) ## covert output to one hot one_hot = np.zeros((y_pred_val.shape[0], 4)) one_hot[np.arange(y_pred_val.shape[0]), y_pred_val] = 1 ## call the scorer acc, microPrecision, microRecall, microF1 = getMetrics(one_hot, y_val, verbose=True) txtfile = open(txt_file, 'a') txtfile.write("(EXPERIMENT %d) microF1 score %f" % ((i + 1), microF1)) txtfile.write("\n--------------------\n") txtfile.close() result = [i, microF1] with open(constant.save_path + record_file, 'a') as f: writer = csv.writer(f) writer.writerow(result) microF1s = microF1s + microF1 microF1s = microF1s / constant.num_split
def main(): args = constant.arg if not os.path.exists(constant.save_path): os.makedirs(constant.save_path) #device = torch.device("cuda", 3) tokenizer = BertTokenizer.from_pretrained('bert-base-cased') f1_avg = [] for seed in range(10): train, val, val_nolab, emoji_tokens, emoji_vectors = get_data_for_bert( seed=seed, emoji_dim=args.emoji_dim) train_emojis, val_emojis, test_emojis = emoji_tokens train_examples = read_examples(train) val_examples = read_examples(val) if args.hier: max_seq_length = 40 else: max_seq_length = 100 train_features = convert_examples_to_features( examples=train_examples, seq_length=max_seq_length, tokenizer=tokenizer, hier=args.hier) val_features = convert_examples_to_features(examples=val_examples, seq_length=max_seq_length, tokenizer=tokenizer, hier=args.hier) if args.hier: model = HierBertModel( context_encoder=args.context_encoder, dropout=args.dropout, double_supervision=args.double_supervision, emoji_vectors=emoji_vectors if args.emoji_emb else None) else: model = FlatBertModel() criterion = nn.CrossEntropyLoss() model.cuda() # Prepare optimizer if args.use_bertadam: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [ p for n, p in param_optimizer if not any(nd in n for nd in no_decay) ], 'weight_decay': 0.01 }, { 'params': [ p for n, p in param_optimizer if any(nd in n for nd in no_decay) ], 'weight_decay': 0.0 }] optimizer = BertAdam( optimizer_grouped_parameters, lr=5e-5, warmup=0.02, t_total=int(len(train_examples) / args.batch_size / 1 * 15)) elif args.noam: optimizer = NoamOpt( constant.emb_dim, 1, 4000, torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0, betas=(0.9, 0.98), eps=1e-9), ) else: optimizer = Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) #training logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.batch_size) #=====================training dataloader======================== all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor( [f.input_type_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) all_emoji_tokens = torch.tensor([emojis for emojis in train_emojis], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, all_emoji_tokens) train_sampler = RandomSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.batch_size) #=====================val dataloader======================== all_input_ids = torch.tensor([f.input_ids for f in val_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in val_features], dtype=torch.long) all_segment_ids = torch.tensor( [f.input_type_ids for f in val_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in val_features], dtype=torch.long) all_emoji_tokens = torch.tensor([emojis for emojis in val_emojis], dtype=torch.long) val_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, all_emoji_tokens) val_sampler = SequentialSampler(val_data) val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=args.batch_size) best_f1 = 0 early_stop = 0 for _ in trange(100, desc="Epoch"): model.train() tr_loss = 0 nb_tr_steps = 0 for step, batch in enumerate( tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.cuda() for t in batch) input_ids, input_mask, segment_ids, label_ids, emoji_tokens = batch #print(input_ids.size()) logits = model( input_ids, segment_ids, input_mask, args.sum_tensor, train=True, emoji_tokens=emoji_tokens if args.emoji_emb else None, last_hidden=args.last_hidden) #print(logits.size(), label_ids.size()) if len(logits) == 2: loss = (1 - args.super_ratio) * criterion( logits[0], label_ids) + args.super_ratio * criterion( logits[1], label_ids) else: loss = criterion(logits, label_ids) loss.backward() tr_loss += loss.item() nb_tr_steps += 1 optimizer.step() model.zero_grad() logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(val_examples)) logger.info(" Batch size = %d", args.batch_size) model.eval() all_logits = [] all_labels = [] for step, batch in enumerate(tqdm(val_dataloader, desc="Iteration")): batch = tuple(t.cuda() for t in batch) input_ids, input_mask, segment_ids, label_ids, emoji_tokens = batch logits = model( input_ids, segment_ids, input_mask, args.sum_tensor, emoji_tokens=emoji_tokens if args.emoji_emb else None, last_hidden=args.last_hidden) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() all_logits.append(logits) all_labels.append(label_ids) accuracy, microPrecision, microRecall, microF1 = getMetrics( np.concatenate(all_logits), np.concatenate(all_labels), verbose=True) if best_f1 < microF1: best_f1 = microF1 save_model(model, seed) else: early_stop += 1 if early_stop > 5: break print('EXPERIMENT:{}, best_f1:{}'.format(seed, best_f1)) f1_avg.append(best_f1) file_summary = constant.save_path + "summary.txt" with open(file_summary, 'w') as the_file: header = "\t".join( ["SPLIT_{}".format(i) for i, _ in enumerate(f1_avg)]) the_file.write(header + "\tAVG\n") ris = "\t".join(["{:.4f}".format(e) for i, e in enumerate(f1_avg)]) the_file.write(ris + "\t{:.4f}\n".format(np.mean(f1_avg)))
from utils.utils import getMetrics from utils.features import get_feature from utils import constant from models.classifier import get_classifier import numpy as np ''' Try python main_classifier.py --emb_dim 300 ''' train, val, dev_no_lab, vocab = prepare_data_for_feature() ## feature_list: glove emoji elmo bert deepmoji emo2vec ## if you want twitter glove or common glove use ty='twitter' and ty='common' X_train, y_train = get_feature(train, vocab, feature_list=['glove'], mode=['sum'],split="train",ty='common') ## [29010,3,emb_size] 3 is number of sentence X_test, y_test = get_feature(val, vocab, feature_list=['glove'], mode=['sum'],split="valid",ty='common') ## [1150,3,emb_size] model = get_classifier(ty='LR') ## train aval and predict model.fit(X_train.reshape(X_train.shape[0], -1), y_train) ## [29010,3,emb_size] --> [29010, 3 * emb_size] ## validate to validation set y_pred = model.predict(X_test.reshape(X_test.shape[0], -1)) ## covert output to one hot one_hot = np.zeros((y_pred.shape[0], 4)) one_hot[np.arange(y_pred.shape[0]), y_pred] = 1 ## call the scorer getMetrics(one_hot,y_test,verbose=True)
def train( model, data_loader_train, data_loader_val, data_loader_test, vocab, patient=10, split=0, verbose=True, ): """ Training loop Inputs: model: the model to be trained data_loader_train: training data loader data_loader_val: validation data loader vocab: vocabulary list Output: avg_best: best f1 score on validation data """ if constant.USE_CUDA: device = torch.device("cuda:{}".format(constant.device)) model.to(device) criterion = nn.CrossEntropyLoss() if constant.noam: opt = NoamOpt( constant.emb_dim, 1, 4000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9), ) else: opt = torch.optim.Adam(model.parameters(), lr=constant.lr) ## TRAINING LOOP avg_best = 0 cnt = 0 for e in range(constant.max_epochs): model.train() loss_log = [] f1_log = [] pbar = tqdm(enumerate(data_loader_train), total=len(data_loader_train)) for i, (X_1, X_2, X_3, x1_len, x2_len, x3_len, y, ind, X_text) in pbar: if constant.noam: opt.optimizer.zero_grad() else: opt.zero_grad() if x1_len is None: pred_prob = model(X_1, X_2, X_3) else: pred_prob = model(X_1, X_2, X_3, x1_len, x2_len, x3_len) if constant.double_supervision: loss = (1 - constant.super_ratio) * criterion( pred_prob[0], y) + constant.super_ratio * criterion( pred_prob[2], y) else: loss = criterion(pred_prob[0], y) if constant.act: R_t = pred_prob[2][0] N_t = pred_prob[2][1] p_t = R_t + N_t avg_p_t = torch.sum( torch.sum(p_t, dim=1) / p_t.size(1)) / p_t.size(0) loss += constant.act_loss_weight * avg_p_t.item() loss.backward() opt.step() ## logging loss_log.append(loss.item()) accuracy, microPrecision, microRecall, microF1 = getMetrics( pred_prob[0].detach().cpu().numpy(), y.cpu().numpy()) f1_log.append(microF1) pbar.set_description( "(Epoch {}) TRAIN MICRO:{:.4f} TRAIN LOSS:{:.4f}".format( (e + 1), np.mean(f1_log), np.mean(loss_log))) ## LOG if e % 1 == 0: microF1 = evaluate(model, criterion, data_loader_val, verbose) if microF1 > avg_best: avg_best = microF1 save_model(model, split) predict( model, criterion, data_loader_test, split) ## print the prediction with the highest Micro-F1 cnt = 0 else: cnt += 1 if cnt == patient: break if avg_best == 1.0: break correct = 0 loss_nb = 0 return avg_best
from utils import constant pred_file_path = constant.pred_file_path ground_file_path = constant.ground_file_path emotion2label = {"others":0, "happy":1, "sad":2, "angry":3} label2emotion = {0:"others", 1:"happy", 2: "sad", 3:"angry"} def read_prediction(file_path): preds = [] with open(file_path, "r") as read_file: for line in read_file: # print(line.replace("\n","")) _, _, _, _, label = line.replace("\n", "").split("\t") if label in emotion2label: preds.append(np.array(emotion2label[label])) return np.array(preds) pred = read_prediction(pred_file_path) one_hot = np.zeros((pred.shape[0], 4)) one_hot[np.arange(pred.shape[0]), pred] = 1 pred = one_hot ground = read_prediction(ground_file_path) print(pred, ground) print(pred.shape, ground.shape) accuracy, microPrecision, microRecall, microF1 = getMetrics(pred, ground,True) print(microF1)
def train(model, data_loader_train, data_loader_val, data_loader_test, vocab, patient=10, split=0): """ Training loop Inputs: model: the model to be trained data_loader_train: training data loader data_loader_val: validation data loader vocab: vocabulary list Output: avg_best: best f1 score on validation data """ if (constant.USE_CUDA): model.cuda() criterion = nn.CrossEntropyLoss() if (constant.noam): opt = NoamOpt( constant.emb_dim, 1, 4000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) else: opt = torch.optim.Adam(model.parameters(), lr=constant.lr) avg_best = 0 cnt = 0 for e in range(constant.max_epochs): model.train() loss_log = [] f1_log = 0 pbar = tqdm(enumerate(data_loader_train), total=len(data_loader_train)) for i, (X, x_len, y, ind, X_text) in pbar: if constant.noam: opt.optimizer.zero_grad() else: opt.zero_grad() if x_len is None: pred_prob = model(X) else: pred_prob = model(X, x_len) loss = criterion(pred_prob[0], y) loss.backward() opt.step() ## logging loss_log.append(loss.item()) accuracy, microPrecision, microRecall, microF1 = getMetrics( pred_prob[0].detach().cpu().numpy(), y.cpu().numpy()) f1_log += microF1 pbar.set_description( "(Epoch {}) TRAIN MICRO:{:.4f} TRAIN LOSS:{:.4f}".format( (e + 1), f1_log / float(i + 1), np.mean(loss_log))) ## LOG if (e % 1 == 0): microF1 = evaluate(model, criterion, data_loader_val) if (microF1 > avg_best): avg_best = microF1 save_model(model, split) predict(model, criterion, data_loader_test, "", split=split ) ## print the prediction with the highest Micro-F1 cnt = 0 else: cnt += 1 if (cnt == patient): break if (avg_best == 1.0): break correct = 0 loss_nb = 0 return avg_best