def test_a_epoch(name, data, model, result_file, label2id, lr, batch_size, en):
model.training = False
full_img_paths = []
evaluator = Evaluator(name, label2id)
for images, labels, img_paths in tqdm(data,
desc=name,
total=np.math.ceil(len(data))):
inp = Variable(images.cuda())
seq_out = model(inp)
pred = argmax(seq_out)
evaluator.append_data(0, pred, labels)
# print(predicted)
full_img_paths.extend(img_paths)
evaluator.gen_results()
evaluator.print_results()
evaluator.write_results(
result_file, "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(
lr, batch_size, en))
return evaluator, full_img_paths
def train_a_epoch(name, data, model, optimizer, criterion, res_file, lr,
batch_size, label2id, en):
model = model.train(True)
evaluator = Evaluator(name, label2id)
print("evaluator loaded")
i = 0
pbar = tqdm(data, desc=name, total=np.math.ceil(len(data)))
for images, labels, _ in pbar:
# zero the parameter gradients
sys.stdout.flush()
optimizer.zero_grad()
if i == 0:
pbar.total = np.math.ceil(len(data))
inp = Variable(images.cuda())
seq_out = model(inp)
pred = argmax(seq_out)
loss = criterion(seq_out, Variable(torch.cuda.LongTensor(labels)))
pbar.set_description("{0} loss: {1}".format(name, to_scalar(loss)))
evaluator.append_data(to_scalar(loss), pred, labels)
loss.backward()
optimizer.step()
i += 1
print("Training Done")
evaluator.gen_results()
evaluator.print_results()
evaluator.write_results(
res_file, "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(
lr, batch_size, en))
return model
def test_a_epoch(name, data, model, result_file, cfg, en):
pred_list = []
true_list = []
evaluator = Evaluator(name,
label2id=cfg['LABEL_2_ID'],
pure_labels=cfg['PURE_LABELS'])
for images, labels in tqdm(data, desc=name, total=np.math.ceil(len(data))):
seq_out = model(Variable(torch.from_numpy(images).float().cuda()))
pred = argmax(seq_out)
evaluator.append_data(0, pred, labels)
# print(predicted)
pred_list.extend(pred)
true_list.extend(labels)
evaluator.gen_results()
evaluator.print_results()
evaluator.write_results(
result_file, "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(
cfg['LEARNING_RATE'], cfg['BATCH_SIZE'], en))
return evaluator, pred_list, true_list
def train_a_epoch(name, data, tag_idx, model, optimizer):
evaluator = Evaluator(name, [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
t = tqdm(data, total=len(data))
for SENT, X, Y, P in t:
# zero the parameter gradients
optimizer.zero_grad()
model.zero_grad()
np.set_printoptions(threshold=np.nan)
nnl = model.neg_log_likelihood(X, Y)
logger.debug("tensor X variable: {0}".format(X))
nnl.backward()
preds = model(X)
for pred, x, y in zip(preds, X, Y):
evaluator.append_data(to_scalar(nnl), pred, x, y)
if cfg.CLIP is not None:
clip_grad_norm(model.parameters(), cfg.CLIP)
optimizer.step()
evaluator.classification_report()
return evaluator, model
def test_a_epoch(name, data, model, result_file, label2id):
model.training = False
pred_list = []
true_list = []
evaluator = Evaluator(name, label2id)
for images, labels in tqdm(data, desc=name, total=np.math.ceil(len(data))):
inp = Variable(images.cuda())
seq_out = model(inp)
pred = argmax(seq_out)
evaluator.append_data(0, pred, labels)
# print(predicted)
pred_list.extend(pred)
true_list.extend(labels)
evaluator.gen_results()
evaluator.print_results()
evaluator.write_results(result_file, '')
return evaluator, pred_list, true_list
def train_a_epoch(name, data, model, optimizer, criterion, res_file, lr,
batch_size, label2id, en):
model.training = True
evaluator = Evaluator(name, label2id)
print("evaluator loaded")
i = 0
for images, labels, _ in tqdm(data,
desc=name,
total=np.math.ceil(len(data))):
# zero the parameter gradients
sys.stdout.flush()
optimizer.zero_grad()
model.zero_grad()
# image = crop(sample.image, cfg.MAX_IMG_SIZE)
inp = Variable(images.cuda())
# inp = torch.transpose(inp, 1, 3)
seq_out = model(inp)
pred = argmax(seq_out)
loss = criterion(seq_out, Variable(torch.cuda.LongTensor(labels)))
evaluator.append_data(to_scalar(loss), pred, labels)
loss.backward()
optimizer.step()
i += 1
print("Training Done")
evaluator.gen_results()
evaluator.print_results()
evaluator.write_results(
res_file, "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(
lr, batch_size, en))
return model
def test(name, data, tag_idx, model):
pred_list = []
true_list = []
full_eval = Evaluator(name, [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
only_ents_eval = Evaluator("test_ents_only", [0, 1],
skip_label=['B-Action', 'I-Action'],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
for SENT, X, Y, P in tqdm(data, desc=name, total=len(data)):
np.set_printoptions(threshold=np.nan)
preds = model(X)
for pred, x, y in zip(preds, X, Y):
full_eval.append_data(0, pred, x, y)
only_ents_eval.append_data(0, pred, x, y)
pred_list.append(pred[1:-1])
true_list.append(y[1:-1])
full_eval.classification_report()
only_ents_eval.classification_report()
return full_eval, pred_list, true_list
def train_a_epoch(name, data, tag_idx, is_oov, model, optimizer, seq_criterion,
lm_f_criterion, lm_b_criterion, att_loss, gamma):
evaluator = Evaluator(name, [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
t = tqdm(data, total=len(data))
if is_oov[0] == 1:
print("Yes, UNKNOWN token is out of vocab")
else:
print("No, UNKNOWN token is not out of vocab")
for SENT, X, C, POS, Y, P in t:
batch_size = len(SENT)
# zero the parameter gradients
optimizer.zero_grad()
model.zero_grad()
model.init_state(len(X))
x_var, c_var, pos_var, y_var, lm_X = to_variables(X=X,
C=C,
POS=POS,
Y=Y)
np.set_printoptions(threshold=np.nan)
if cfg.CHAR_LEVEL == "Attention":
lm_f_out, lm_b_out, seq_out, seq_lengths, emb, char_emb = model(
x_var, c_var)
unrolled_x_var = list(chain.from_iterable(x_var))
not_oov_seq = [-1 if is_oov[idx] else 1 for idx in unrolled_x_var]
char_att_loss = att_loss(
emb.detach(), char_emb,
Variable(torch.cuda.LongTensor(not_oov_seq))) / batch_size
else:
lm_f_out, lm_b_out, seq_out, seq_lengths = model(x_var, c_var)
logger.debug("lm_f_out : {0}".format(lm_f_out))
logger.debug("lm_b_out : {0}".format(lm_b_out))
logger.debug("seq_out : {0}".format(seq_out))
logger.debug("tensor X variable: {0}".format(x_var))
# remove start and stop tags
pred = argmax(seq_out)
logger.debug("Predicted output {0}".format(pred))
seq_loss = seq_criterion(
seq_out,
Variable(torch.LongTensor(y_var)).cuda()) / batch_size
# to limit the vocab size of the sample sentence ( trick used to improve lm model)
# TODO make sure that start and end symbol of sentence gets through this filtering.
logger.debug("Sample input {0}".format(lm_X))
if gamma != 0:
lm_X_f = [x1d[1:] for x1d in lm_X]
lm_X_b = [x1d[:-1] for x1d in lm_X]
lm_X_f = list(chain.from_iterable(lm_X_f))
lm_X_b = list(chain.from_iterable(lm_X_b))
lm_f_loss = lm_f_criterion(
lm_f_out.squeeze(),
Variable(cuda.LongTensor(lm_X_f)).squeeze()) / batch_size
lm_b_loss = lm_b_criterion(
lm_b_out.squeeze(),
Variable(cuda.LongTensor(lm_X_b)).squeeze()) / batch_size
if cfg.CHAR_LEVEL == "Attention":
total_loss = seq_loss + Variable(cuda.FloatTensor(
[gamma])) * (lm_f_loss + lm_b_loss) + char_att_loss
else:
total_loss = seq_loss + Variable(cuda.FloatTensor(
[gamma])) * (lm_f_loss + lm_b_loss)
else:
if cfg.CHAR_LEVEL == "Attention":
total_loss = seq_loss + char_att_loss
else:
total_loss = seq_loss
desc = "total_loss: {0:.4f} = seq_loss: {1:.4f}".format(
to_scalar(total_loss), to_scalar(seq_loss))
if gamma != 0:
desc += " + gamma: {0} * (lm_f_loss: {1:.4f} + lm_b_loss: {2:.4f})".format(
gamma, to_scalar(lm_f_loss), to_scalar(lm_b_loss))
if cfg.CHAR_LEVEL == "Attention":
desc += " + char_att_loss: {0:.4f}".format(
to_scalar(char_att_loss))
t.set_description(desc)
preds = roll(pred, seq_lengths)
for pred, x, y in zip(preds, X, Y):
evaluator.append_data(to_scalar(total_loss), pred, x, y)
total_loss.backward()
if cfg.CLIP is not None:
clip_grad_norm(model.parameters(), cfg.CLIP)
optimizer.step()
evaluator.classification_report()
return evaluator, model
def test(name, data, tag_idx, model):
correct = 0
total = 0
pred_list = []
true_list = []
full_eval = Evaluator(name, [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
only_ents_eval = Evaluator("test_ents_only", [0, 1],
skip_label=['B-Action', 'I-Action'],
main_label_name=cfg.POSITIVE_LABEL,
label2id=tag_idx,
conll_eval=True)
for SENT, X, C, POS, Y, P in tqdm(data, desc=name, total=len(data)):
np.set_printoptions(threshold=np.nan)
model.init_state(len(X))
x_var, c_var, pos_var, y_var, lm_X = to_variables(X=X,
C=C,
POS=POS,
Y=Y)
if cfg.CHAR_LEVEL == "Attention":
lm_f_out, lm_b_out, seq_out, seq_lengths, emb, char_emb = model(
x_var, c_var)
else:
lm_f_out, lm_b_out, seq_out, seq_lengths = model(x_var, c_var)
pred = argmax(seq_out)
preds = roll(pred, seq_lengths)
for pred, x, y in zip(preds, X, Y):
full_eval.append_data(0, pred, x, y)
only_ents_eval.append_data(0, pred, x, y)
pred_list.append(pred[1:-1])
true_list.append(y[1:-1])
full_eval.classification_report()
only_ents_eval.classification_report()
return full_eval, only_ents_eval, pred_list, true_list
def main():
dataset = dataset_prep(loadfile=cfg.DB_MAXENT)
dataset.tag_idx['<s>'] = len(dataset.tag_idx.keys())
dataset.tag_idx['</s>'] = len(dataset.tag_idx.keys())
total = len(dataset.tokens2d)
print(dataset.tag_idx)
print(total)
print(len(dataset.cut_list))
ntrain = int(total * .60)
ndev = int(total * .80)
ntest = total
ablation = [
['pos', 'ng0', 'bg', 'rel', 'dep', 'gov', 'lm'], # FULL
['ng0', 'bg', 'rel', 'dep', 'gov', 'lm'], # -POS
['pos', 'rel', 'dep', 'gov', 'lm'], # -UNIGRAM/BIGRAM
['pos', 'ng0', 'bg', 'lm'], # -DEPs
['pos', 'ng0', 'bg', 'rel', 'dep', 'gov'], # -LEMMA
]
addition = [['pos'], ['ng0', 'bg'], ['ng0', 'bg', 'lm'],
['pos', 'ng0', 'bg'], ['pos', 'ng0', 'bg', 'lm'],
['pos', 'ng0', 'bg', 'rel', 'dep', 'gov', 'lm']]
for feat in addition:
x_train, w_train, y_train = extract_data(0, ntrain, dataset, feat)
# print(list(dataset.f_df.columns.values))
x_test, w_test, y_test = extract_data(ndev, ntest, dataset, feat)
model = LogisticRegression(solver='lbfgs',
multi_class='multinomial',
n_jobs=8)
model.fit(x_train, y_train)
pred = model.predict(x_test)
print("ALL!")
evaluator = Evaluator("test_all", [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=dataset.tag_idx,
conll_eval=True)
evaluator.append_data(0, pred, w_test, y_test)
evaluator.classification_report()
print("ONLY ENTITIES!")
evaluator = Evaluator("test_ents_only", [0, 1],
skip_label=['B-Action', 'I-Action'],
main_label_name=cfg.POSITIVE_LABEL,
label2id=dataset.tag_idx,
conll_eval=True)
evaluator.append_data(0, pred, w_test, y_test)
evaluator.classification_report()
def pos():
dataset = dataset_prep(loadfile=cfg.DB_MAXENT)
dataset.tag_idx['<s>'] = len(dataset.tag_idx.keys())
dataset.tag_idx['</s>'] = len(dataset.tag_idx.keys())
# dataset.pos_table(["B-Action", "I-Action"], ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'NN', 'NNP', 'NNS', 'JJ'])
dataset.ent_table()
total = len(dataset.tokens2d)
print(dataset.tag_idx)
print(total)
print(len(dataset.cut_list))
ntrain = int(total * .60)
ndev = int(total * .80)
ntest = total
x_train, w_train, y_train = pos_verb_only(
0, ntrain, dataset, ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'])
# print(list(dataset.f_df.columns.values))
x_test, w_test, y_test = pos_verb_only(
ndev, ntest, dataset, ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'])
model = LogisticRegression(solver='lbfgs',
multi_class='multinomial',
n_jobs=8)
model.fit(x_train, y_train)
pred = model.predict(x_test)
print("ALL!")
evaluator = Evaluator("test_all", [0, 1],
main_label_name=cfg.POSITIVE_LABEL,
label2id=dataset.tag_idx,
conll_eval=True)
evaluator.append_data(0, pred, w_test, y_test)
evaluator.classification_report()
print("ONLY ENTITIES!")
evaluator = Evaluator("test_ents_only", [0, 1],
skip_label=['B-Action', 'I-Action'],
main_label_name=cfg.POSITIVE_LABEL,
label2id=dataset.tag_idx,
conll_eval=True)
evaluator.append_data(0, pred, w_test, y_test)
evaluator.classification_report()