def evaluate(model, criterion, loader, verbose=True):
model.eval()
pred = []
gold = []
for X_1, X_2, X_3, x1_len, x2_len, x3_len, y, ind, X_text in loader:
if x1_len is None:
pred_prob = model(X_1, X_2, X_3)
else:
pred_prob = model(X_1, X_2, X_3, x1_len, x2_len, x3_len)
# pred_prob = model(X_1,X_2,X_3,x1_len,x2_len,x3_len)
pred.append(pred_prob[0].detach().cpu().numpy())
gold.append(y.cpu().numpy())
pred = np.concatenate(pred)
gold = np.concatenate(gold)
accuracy, microPrecision, microRecall, microF1 = getMetrics(
pred, gold, verbose)
return microF1
if i == 0:
txtfile = open(txt_file, 'a')
txtfile.write("\n--------------------\n")
txtfile.write("Classifier %s, Parameters: %f, %f, %f" %
(classifier, parameter_list[0], parameter_list[1],
parameter_list[2]))
txtfile.close()
y_pred_val = model.predict(X_val.reshape(X_val.shape[0], -1))
## covert output to one hot
one_hot = np.zeros((y_pred_val.shape[0], 4))
one_hot[np.arange(y_pred_val.shape[0]), y_pred_val] = 1
## call the scorer
acc, microPrecision, microRecall, microF1 = getMetrics(one_hot,
y_val,
verbose=True)
txtfile = open(txt_file, 'a')
txtfile.write("(EXPERIMENT %d) microF1 score %f" % ((i + 1), microF1))
txtfile.write("\n--------------------\n")
txtfile.close()
result = [i, microF1]
with open(constant.save_path + record_file, 'a') as f:
writer = csv.writer(f)
writer.writerow(result)
microF1s = microF1s + microF1
microF1s = microF1s / constant.num_split
def main():
args = constant.arg
if not os.path.exists(constant.save_path):
os.makedirs(constant.save_path)
#device = torch.device("cuda", 3)
tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
f1_avg = []
for seed in range(10):
train, val, val_nolab, emoji_tokens, emoji_vectors = get_data_for_bert(
seed=seed, emoji_dim=args.emoji_dim)
train_emojis, val_emojis, test_emojis = emoji_tokens
train_examples = read_examples(train)
val_examples = read_examples(val)
if args.hier:
max_seq_length = 40
else:
max_seq_length = 100
train_features = convert_examples_to_features(
examples=train_examples,
seq_length=max_seq_length,
tokenizer=tokenizer,
hier=args.hier)
val_features = convert_examples_to_features(examples=val_examples,
seq_length=max_seq_length,
tokenizer=tokenizer,
hier=args.hier)
if args.hier:
model = HierBertModel(
context_encoder=args.context_encoder,
dropout=args.dropout,
double_supervision=args.double_supervision,
emoji_vectors=emoji_vectors if args.emoji_emb else None)
else:
model = FlatBertModel()
criterion = nn.CrossEntropyLoss()
model.cuda()
# Prepare optimizer
if args.use_bertadam:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params': [
p for n, p in param_optimizer
if any(nd in n for nd in no_decay)
],
'weight_decay':
0.0
}]
optimizer = BertAdam(
optimizer_grouped_parameters,
lr=5e-5,
warmup=0.02,
t_total=int(len(train_examples) / args.batch_size / 1 * 15))
elif args.noam:
optimizer = NoamOpt(
constant.emb_dim,
1,
4000,
torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=0,
betas=(0.9, 0.98),
eps=1e-9),
)
else:
optimizer = Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=1e-3)
#training
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.batch_size)
#=====================training dataloader========================
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor(
[f.input_type_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
all_emoji_tokens = torch.tensor([emojis for emojis in train_emojis],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids,
all_emoji_tokens)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.batch_size)
#=====================val dataloader========================
all_input_ids = torch.tensor([f.input_ids for f in val_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in val_features],
dtype=torch.long)
all_segment_ids = torch.tensor(
[f.input_type_ids for f in val_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in val_features],
dtype=torch.long)
all_emoji_tokens = torch.tensor([emojis for emojis in val_emojis],
dtype=torch.long)
val_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids,
all_emoji_tokens)
val_sampler = SequentialSampler(val_data)
val_dataloader = DataLoader(val_data,
sampler=val_sampler,
batch_size=args.batch_size)
best_f1 = 0
early_stop = 0
for _ in trange(100, desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_steps = 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.cuda() for t in batch)
input_ids, input_mask, segment_ids, label_ids, emoji_tokens = batch
#print(input_ids.size())
logits = model(
input_ids,
segment_ids,
input_mask,
args.sum_tensor,
train=True,
emoji_tokens=emoji_tokens if args.emoji_emb else None,
last_hidden=args.last_hidden)
#print(logits.size(), label_ids.size())
if len(logits) == 2:
loss = (1 - args.super_ratio) * criterion(
logits[0], label_ids) + args.super_ratio * criterion(
logits[1], label_ids)
else:
loss = criterion(logits, label_ids)
loss.backward()
tr_loss += loss.item()
nb_tr_steps += 1
optimizer.step()
model.zero_grad()
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(val_examples))
logger.info(" Batch size = %d", args.batch_size)
model.eval()
all_logits = []
all_labels = []
for step, batch in enumerate(tqdm(val_dataloader,
desc="Iteration")):
batch = tuple(t.cuda() for t in batch)
input_ids, input_mask, segment_ids, label_ids, emoji_tokens = batch
logits = model(
input_ids,
segment_ids,
input_mask,
args.sum_tensor,
emoji_tokens=emoji_tokens if args.emoji_emb else None,
last_hidden=args.last_hidden)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
all_logits.append(logits)
all_labels.append(label_ids)
accuracy, microPrecision, microRecall, microF1 = getMetrics(
np.concatenate(all_logits),
np.concatenate(all_labels),
verbose=True)
if best_f1 < microF1:
best_f1 = microF1
save_model(model, seed)
else:
early_stop += 1
if early_stop > 5:
break
print('EXPERIMENT:{}, best_f1:{}'.format(seed, best_f1))
f1_avg.append(best_f1)
file_summary = constant.save_path + "summary.txt"
with open(file_summary, 'w') as the_file:
header = "\t".join(
["SPLIT_{}".format(i) for i, _ in enumerate(f1_avg)])
the_file.write(header + "\tAVG\n")
ris = "\t".join(["{:.4f}".format(e) for i, e in enumerate(f1_avg)])
the_file.write(ris + "\t{:.4f}\n".format(np.mean(f1_avg)))
from utils.utils import getMetrics from utils.features import get_feature from utils import constant from models.classifier import get_classifier import numpy as np ''' Try python main_classifier.py --emb_dim 300 ''' train, val, dev_no_lab, vocab = prepare_data_for_feature() ## feature_list: glove emoji elmo bert deepmoji emo2vec ## if you want twitter glove or common glove use ty='twitter' and ty='common' X_train, y_train = get_feature(train, vocab, feature_list=['glove'], mode=['sum'],split="train",ty='common') ## [29010,3,emb_size] 3 is number of sentence X_test, y_test = get_feature(val, vocab, feature_list=['glove'], mode=['sum'],split="valid",ty='common') ## [1150,3,emb_size] model = get_classifier(ty='LR') ## train aval and predict model.fit(X_train.reshape(X_train.shape[0], -1), y_train) ## [29010,3,emb_size] --> [29010, 3 * emb_size] ## validate to validation set y_pred = model.predict(X_test.reshape(X_test.shape[0], -1)) ## covert output to one hot one_hot = np.zeros((y_pred.shape[0], 4)) one_hot[np.arange(y_pred.shape[0]), y_pred] = 1 ## call the scorer getMetrics(one_hot,y_test,verbose=True)
def train(
model,
data_loader_train,
data_loader_val,
data_loader_test,
vocab,
patient=10,
split=0,
verbose=True,
):
"""
Training loop
Inputs:
model: the model to be trained
data_loader_train: training data loader
data_loader_val: validation data loader
vocab: vocabulary list
Output:
avg_best: best f1 score on validation data
"""
if constant.USE_CUDA:
device = torch.device("cuda:{}".format(constant.device))
model.to(device)
criterion = nn.CrossEntropyLoss()
if constant.noam:
opt = NoamOpt(
constant.emb_dim,
1,
4000,
torch.optim.Adam(model.parameters(),
lr=0,
betas=(0.9, 0.98),
eps=1e-9),
)
else:
opt = torch.optim.Adam(model.parameters(), lr=constant.lr)
## TRAINING LOOP
avg_best = 0
cnt = 0
for e in range(constant.max_epochs):
model.train()
loss_log = []
f1_log = []
pbar = tqdm(enumerate(data_loader_train), total=len(data_loader_train))
for i, (X_1, X_2, X_3, x1_len, x2_len, x3_len, y, ind, X_text) in pbar:
if constant.noam:
opt.optimizer.zero_grad()
else:
opt.zero_grad()
if x1_len is None:
pred_prob = model(X_1, X_2, X_3)
else:
pred_prob = model(X_1, X_2, X_3, x1_len, x2_len, x3_len)
if constant.double_supervision:
loss = (1 - constant.super_ratio) * criterion(
pred_prob[0], y) + constant.super_ratio * criterion(
pred_prob[2], y)
else:
loss = criterion(pred_prob[0], y)
if constant.act:
R_t = pred_prob[2][0]
N_t = pred_prob[2][1]
p_t = R_t + N_t
avg_p_t = torch.sum(
torch.sum(p_t, dim=1) / p_t.size(1)) / p_t.size(0)
loss += constant.act_loss_weight * avg_p_t.item()
loss.backward()
opt.step()
## logging
loss_log.append(loss.item())
accuracy, microPrecision, microRecall, microF1 = getMetrics(
pred_prob[0].detach().cpu().numpy(),
y.cpu().numpy())
f1_log.append(microF1)
pbar.set_description(
"(Epoch {}) TRAIN MICRO:{:.4f} TRAIN LOSS:{:.4f}".format(
(e + 1), np.mean(f1_log), np.mean(loss_log)))
## LOG
if e % 1 == 0:
microF1 = evaluate(model, criterion, data_loader_val, verbose)
if microF1 > avg_best:
avg_best = microF1
save_model(model, split)
predict(
model, criterion, data_loader_test,
split) ## print the prediction with the highest Micro-F1
cnt = 0
else:
cnt += 1
if cnt == patient:
break
if avg_best == 1.0:
break
correct = 0
loss_nb = 0
return avg_best
from utils import constant
pred_file_path = constant.pred_file_path
ground_file_path = constant.ground_file_path
emotion2label = {"others":0, "happy":1, "sad":2, "angry":3}
label2emotion = {0:"others", 1:"happy", 2: "sad", 3:"angry"}
def read_prediction(file_path):
preds = []
with open(file_path, "r") as read_file:
for line in read_file:
# print(line.replace("\n",""))
_, _, _, _, label = line.replace("\n", "").split("\t")
if label in emotion2label:
preds.append(np.array(emotion2label[label]))
return np.array(preds)
pred = read_prediction(pred_file_path)
one_hot = np.zeros((pred.shape[0], 4))
one_hot[np.arange(pred.shape[0]), pred] = 1
pred = one_hot
ground = read_prediction(ground_file_path)
print(pred, ground)
print(pred.shape, ground.shape)
accuracy, microPrecision, microRecall, microF1 = getMetrics(pred, ground,True)
print(microF1)
def train(model,
data_loader_train,
data_loader_val,
data_loader_test,
vocab,
patient=10,
split=0):
"""
Training loop
Inputs:
model: the model to be trained
data_loader_train: training data loader
data_loader_val: validation data loader
vocab: vocabulary list
Output:
avg_best: best f1 score on validation data
"""
if (constant.USE_CUDA): model.cuda()
criterion = nn.CrossEntropyLoss()
if (constant.noam):
opt = NoamOpt(
constant.emb_dim, 1, 4000,
torch.optim.Adam(model.parameters(),
lr=0,
betas=(0.9, 0.98),
eps=1e-9))
else:
opt = torch.optim.Adam(model.parameters(), lr=constant.lr)
avg_best = 0
cnt = 0
for e in range(constant.max_epochs):
model.train()
loss_log = []
f1_log = 0
pbar = tqdm(enumerate(data_loader_train), total=len(data_loader_train))
for i, (X, x_len, y, ind, X_text) in pbar:
if constant.noam:
opt.optimizer.zero_grad()
else:
opt.zero_grad()
if x_len is None: pred_prob = model(X)
else: pred_prob = model(X, x_len)
loss = criterion(pred_prob[0], y)
loss.backward()
opt.step()
## logging
loss_log.append(loss.item())
accuracy, microPrecision, microRecall, microF1 = getMetrics(
pred_prob[0].detach().cpu().numpy(),
y.cpu().numpy())
f1_log += microF1
pbar.set_description(
"(Epoch {}) TRAIN MICRO:{:.4f} TRAIN LOSS:{:.4f}".format(
(e + 1), f1_log / float(i + 1), np.mean(loss_log)))
## LOG
if (e % 1 == 0):
microF1 = evaluate(model, criterion, data_loader_val)
if (microF1 > avg_best):
avg_best = microF1
save_model(model, split)
predict(model, criterion, data_loader_test, "", split=split
) ## print the prediction with the highest Micro-F1
cnt = 0
else:
cnt += 1
if (cnt == patient): break
if (avg_best == 1.0): break
correct = 0
loss_nb = 0
return avg_best
