def main(args):
model = utils.get_models(bert_config=args.bert_config,
pred_n_labels=args.pred_n_labels,
arg_n_labels=args.arg_n_labels,
n_arg_heads=args.n_arg_heads,
n_arg_layers=args.n_arg_layers,
pos_emb_dim=args.pos_emb_dim,
use_lstm=args.use_lstm,
device=args.device)
if torch.cuda.is_available():
map_location = lambda storage, loc: storage.cuda()
else:
map_location = 'cpu'
model.load_state_dict(
torch.load(args.model_path, map_location=map_location))
model.zero_grad()
model.eval()
loader = load_data(data_path=args.test_data_path,
batch_size=args.batch_size,
tokenizer_config=args.bert_config,
train=False)
start = time.time()
extract(args, model, loader, args.save_path)
print("TIME: ", time.time() - start)
test_results = do_eval(args.save_path, args.test_gold_path)
utils.print_results("TEST RESULT", test_results,
["F1 ", "PREC", "REC ", "AUC "])
def video_process(video_path, pm_model, save_video_flag):
video_thread = VideoThread(video_path, 1280, 960, 1, '视频线程')
video_thread.start()
serial_thread = SerialThread('串口线程')
serial_thread.start()
init_flag = True
while True:
frame_read = video_thread.get_image()
if frame_read is None:
print('获取视频失败！')
break
# if init_flag and save_video_flag:
# # 视频模式输出检测视频
# save_name = 'save_video.avi'
# print('保存视频到' + save_name)
# out_video = cv2.VideoWriter(save_name, cv2.VideoWriter_fourcc(*"MJPG"), 10.0,
# (frame_read.shape[1], frame_read.shape[0]))
# init_flag = False
if init_flag:
init_flag = False
continue
# [类别编号, 置信度, 中点坐标, 左上坐标, 右下坐标]
boxes = pm_model.predict(frame_read)
print_results(boxes, pm_model.label_names, init_flag)
draw_results(frame_read, boxes, pm_model.colors, pm_model.label_names,
False)
serial_thread.set_data(boxes)
def apply_calculations(options, set_points, test_points):
dimension = len(set_points[0]) - 1
random_polynomials = utils.generate_random_polynomials(dimension, options.max_polynomial_degree)
set_points_min_max = utils.get_column_max_min(set_points)
scaled_set_points = utils.scale_points(set_points, set_points_min_max)
scaled_test_points = utils.scale_points(test_points, set_points_min_max)
best_model = find_best_model(options, scaled_set_points, random_polynomials)
best_model_with_best_lmbda = find_best_lmbda_for_model(options, best_model, random_polynomials, scaled_set_points,
scaled_test_points, set_points_min_max)
utils.print_results(best_model_with_best_lmbda)
def main(args):
utils.set_seed(args.seed)
model = utils.get_models(bert_config=args.bert_config,
pred_n_labels=args.pred_n_labels,
arg_n_labels=args.arg_n_labels,
n_arg_heads=args.n_arg_heads,
n_arg_layers=args.n_arg_layers,
lstm_dropout=args.lstm_dropout,
mh_dropout=args.mh_dropout,
pred_clf_dropout=args.pred_clf_dropout,
arg_clf_dropout=args.arg_clf_dropout,
pos_emb_dim=args.pos_emb_dim,
use_lstm=args.use_lstm,
device=args.device)
trn_loader = load_data(data_path=args.trn_data_path,
batch_size=args.batch_size,
max_len=args.max_len,
tokenizer_config=args.bert_config)
dev_loaders = [
load_data(data_path=cur_dev_path,
batch_size=args.dev_batch_size,
tokenizer_config=args.bert_config,
train=False) for cur_dev_path in args.dev_data_path
]
args.total_steps = round(len(trn_loader) * args.epochs)
args.warmup_steps = round(args.total_steps / 10)
optimizer, scheduler = utils.get_train_modules(
model=model,
lr=args.learning_rate,
total_steps=args.total_steps,
warmup_steps=args.warmup_steps)
model.zero_grad()
summarizer = SummaryManager(args)
print("\nTraining Starts\n")
for epoch in tqdm(range(1, args.epochs + 1), desc='epochs'):
trn_results = train(args, epoch, model, trn_loader, dev_loaders,
summarizer, optimizer, scheduler)
# extraction on devset
dev_iter = zip(args.dev_data_path, args.dev_gold_path, dev_loaders)
dev_results = list()
total_sum = 0
for dev_input, dev_gold, dev_loader in dev_iter:
dev_name = dev_input.split('/')[-1].replace('.pkl', '')
output_path = os.path.join(
args.save_path, f'epoch{epoch}_dev/end_epoch/{dev_name}')
extract(args, model, dev_loader, output_path)
dev_result = do_eval(output_path, dev_gold)
utils.print_results(f"EPOCH{epoch} EVAL", dev_result,
["F1 ", "PREC", "REC ", "AUC "])
total_sum += dev_result[0] + dev_result[-1]
dev_result.append(dev_result[0] + dev_result[-1])
dev_results += dev_result
summarizer.save_results([epoch] + trn_results + dev_results +
[total_sum])
model_name = utils.set_model_name(total_sum, epoch)
torch.save(model.state_dict(), os.path.join(args.save_path,
model_name))
print("\nTraining Ended\n")
def train(args, epoch, model, trn_loader, dev_loaders, summarizer, optimizer,
scheduler):
total_pred_loss, total_arg_loss, trn_results = 0, 0, None
epoch_steps = int(args.total_steps / args.epochs)
iterator = tqdm(enumerate(trn_loader), desc='steps', total=epoch_steps)
for step, batch in iterator:
batch = map(lambda x: x.to(args.device), batch)
token_ids, att_mask, single_pred_label, single_arg_label, all_pred_label = batch
pred_mask = bio.get_pred_mask(single_pred_label)
model.train()
model.zero_grad()
# feed to predicate model
batch_loss, pred_loss, arg_loss = model(
input_ids=token_ids,
attention_mask=att_mask,
predicate_mask=pred_mask,
total_pred_labels=all_pred_label,
arg_labels=single_arg_label)
# get performance on this batch
total_pred_loss += pred_loss.item()
total_arg_loss += arg_loss.item()
batch_loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
scheduler.step()
trn_results = [
total_pred_loss / (step + 1), total_arg_loss / (step + 1)
]
if step > epoch_steps:
break
# interim evaluation
if step % 1000 == 0 and step != 0:
dev_iter = zip(args.dev_data_path, args.dev_gold_path, dev_loaders)
dev_results = list()
total_sum = 0
for dev_input, dev_gold, dev_loader in dev_iter:
dev_name = dev_input.split('/')[-1].replace('.pkl', '')
output_path = os.path.join(
args.save_path, f'epoch{epoch}_dev/step{step}/{dev_name}')
extract(args, model, dev_loader, output_path)
dev_result = do_eval(output_path, dev_gold)
utils.print_results(f"EPOCH{epoch} STEP{step} EVAL",
dev_result,
["F1 ", "PREC", "REC ", "AUC "])
total_sum += dev_result[0] + dev_result[-1]
dev_result.append(dev_result[0] + dev_result[-1])
dev_results += dev_result
summarizer.save_results([step] + trn_results + dev_results +
[total_sum])
model_name = utils.set_model_name(total_sum, epoch, step)
torch.save(model.state_dict(),
os.path.join(args.save_path, model_name))
if step % args.summary_step == 0 and step != 0:
utils.print_results(f"EPOCH{epoch} STEP{step} TRAIN", trn_results,
["PRED LOSS", "ARG LOSS "])
# end epoch summary
utils.print_results(f"EPOCH{epoch} TRAIN", trn_results,
["PRED LOSS", "ARG LOSS "])
return trn_results
def train_model(model,
criterion,
optimizer,
scheduler,
dataloaders,
dataset_sizes,
model_path,
device,
fold_name,
use_gpu=True,
num_epochs=25,
plot_res=True,
predict_test=True,
save_val=True,
best='loss'):
since = time.time()
torch.save(model.state_dict(), os.path.join(model_path, '_temp'))
best_f1 = 0.0
best_loss = np.inf
best_epoch = 0
train_metrics = []
val_metrics = []
print('=' * 10)
print('VAL data {}'.format(fold_name))
print('=' * 10)
for epoch in range(num_epochs):
model.train(True)
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
running_loss = 0.0
pred = np.array([])
true = np.array([])
phase = 'train'
for data in tqdm(dataloaders[phase]):
optimizer.zero_grad()
inputs, labels = data
if use_gpu:
inputs = Variable(inputs).to(device)
labels = Variable(labels).to(device)
else:
inputs, labels = Variable(inputs), Variable(labels)
outputs = model(inputs)
loss = criterion(
outputs.type(torch.cuda.FloatTensor)[:, 1],
labels.type(torch.cuda.FloatTensor))
#loss = nn.BCEWithLogitsLoss(reduction='sum',
# weight=torch.ones(outputs.shape[0]).to(device) / 5)(outputs.type(torch.cuda.FloatTensor)[:, 1],
# labels.type(torch.cuda.FloatTensor))
loss.backward()
optimizer.step()
running_loss += loss.item()
true = np.append(true, to_numpy(labels, use_gpu))
pred = np.append(pred, sigmoid(to_numpy(outputs[:, 1], use_gpu)))
epoch_loss = running_loss / dataset_sizes[phase]
pr_rec_auc, roc_auc, f1_05, f1_max, f1_auc, metr = calc_metrics(
true, pred)
val_true, val_pred, val_loss = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='val')
pr_rec_auc_val, roc_auc_val, f1_05_val, f1_max_val, f1_auc_val, metr_val = calc_metrics(
val_true, val_pred)
if scheduler is not None:
scheduler.step((val_loss) / dataset_sizes['val'])
clear()
clear_output()
print('=' * 10)
print('VAL data {}'.format(fold_name))
print('=' * 10)
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
print_results(phase=phase,
loss=epoch_loss,
roc_auc=roc_auc,
pr_rec_auc=pr_rec_auc,
f1_max=f1_max,
f1_05=f1_05,
f1_auc=f1_auc)
print('\n')
print_results(phase='val',
loss=(val_loss) / dataset_sizes['val'],
roc_auc=roc_auc_val,
pr_rec_auc=pr_rec_auc_val,
f1_max=f1_max_val,
f1_05=f1_05_val,
f1_auc=f1_auc_val)
train_metrics.append(metr + [epoch_loss])
val_metrics.append(metr_val + [(val_loss) / dataset_sizes['val']])
if f1_05_val > best_f1:
best_f1 = f1_05_val
if best == 'metric':
best_epoch = epoch
torch.save(model.state_dict(),
os.path.join(model_path, '_temp'))
if (val_loss) / dataset_sizes['val'] < best_loss:
best_loss = (val_loss) / dataset_sizes['val']
if best == 'loss':
best_epoch = epoch
torch.save(model.state_dict(),
os.path.join(model_path, '_temp'))
time_elapsed = time.time() - since
print('Elapsed {:.0f}m {:.0f}s\n'.format(time_elapsed // 60,
time_elapsed % 60))
np.save(os.path.join(model_path, 'val_metrics_' + fold_name),
val_metrics)
np.save(os.path.join(model_path, 'train_metrics_' + fold_name),
train_metrics)
print('Current best: {:4f}, epoch {}'.format(best_f1, best_epoch))
if plot_res:
vizualize(model_path,
fold_name,
show=False,
save=True,
save_format='.png')
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val f1_05: {:4f}'.format(best_f1))
print('Best val loss: {:4f}'.format(best_loss))
if best is not None:
model.load_state_dict(torch.load(os.path.join(model_path, '_temp')))
vizualize(model_path, fold_name, show=False, save=True, save_format='.pdf')
if predict_test:
results = pd.DataFrame()
test_true, test_pred, _ = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='test')
results['pred'] = test_pred
results['pseudo_true'] = test_true
results['file'] = [
sample[0] for sample in dataloaders['test'].dataset.samples
]
results.to_csv(os.path.join(model_path,
'test_results_' + fold_name + '.csv'),
index=False)
if save_val:
results = pd.DataFrame()
val_true, val_pred, _ = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='val')
results['true'] = val_true
results['pred'] = val_pred
results['file'] = [
sample[0] for sample in dataloaders['val'].dataset.samples
]
results.to_csv(os.path.join(model_path,
'val_results_' + fold_name + '.csv'),
index=False)
del inputs, labels, loss
return model, best_f1
def eval_epoch(self, final=False, save_predictions=False):
"""
Evaluate the model on the test set.
No backward computation is allowed.
"""
t1 = time()
output = {
'tp': [],
'fp': [],
'fn': [],
'tn': [],
'loss': [],
'preds': [],
'truth': [],
'true': 0,
'true_sep': np.zeros(self.rel_size)
}
test_info = []
test_result = []
self.model.eval()
test_iter = self.iterator(self.data['test'],
batch_size=self.params['batch'],
shuffle_=False)
# preds=[]
# truths=[]
for batch_idx, batch in enumerate(test_iter):
batch = self.convert_batch(batch, istrain=False, save=True)
with torch.no_grad():
loss, stats, predictions, select, pred_pairs, multi_truths, mask, _ = self.model(
batch) # pred_pairs <#pair, relations_num>
pred_pairs = torch.sigmoid(pred_pairs)
output['loss'] += [loss.item()]
output['tp'] += [stats['tp'].to('cpu').data.numpy()]
output['fp'] += [stats['fp'].to('cpu').data.numpy()]
output['fn'] += [stats['fn'].to('cpu').data.numpy()]
output['tn'] += [stats['tn'].to('cpu').data.numpy()]
output['preds'] += [predictions.to('cpu').data.numpy()]
# preds.extend(predictions.to('cpu').data.numpy())
# truths.extend(truth.to('cpu').data.numpy())
if True:
test_infos = batch['info'][
select[0].to('cpu').data.numpy(),
select[1].to('cpu').data.numpy(),
select[2].to('cpu').data.numpy()][mask.to(
'cpu').data.numpy()]
test_info += [test_infos]
pred_pairs = pred_pairs.data.cpu().numpy()
multi_truths = multi_truths.data.cpu().numpy()
output['true'] += multi_truths.sum(
) - multi_truths[:, self.loader.label2ignore].sum()
output['true_sep'] = output['true_sep'] + multi_truths.sum(axis=0)
if save_predictions:
assert test_infos.shape[0] == len(pred_pairs), print(
"test info=%d, pred_pair=%d" %
(len(test_infos.shape[0]), len(pred_pairs)))
for pair_id in range(len(pred_pairs)):
multi_truth = multi_truths[pair_id] #第pair_id个实体对的true
for r in range(0, self.rel_size):
if r == self.loader.label2ignore:
continue
test_result.append((int(multi_truth[r]) == 1,
float(pred_pairs[pair_id][r]),
test_infos[pair_id]['intrain'],
test_infos[pair_id]['cross'],
self.loader.index2rel[r], r,
len(test_info) - 1, pair_id))
# estimate performance
total_loss, scores = self.performance(output)
# pairs*rel_size*batch
test_result.sort(key=lambda x: x[1], reverse=True)
input_theta, w, f1, p, r, scores_class = self.tune_f1_theta(
test_result,
output['true'],
output['true_sep'],
self.params['input_theta'],
isTest=save_predictions)
t2 = time()
if not final:
self.test_res['loss'] += [total_loss]
# self.test_res['score'] += [scores[self.primary_metric]]
self.test_res['score'] += [f1]
self.test_res['p'] = p
self.test_res['r'] = r
print(' TEST | LOSS = {:.05f}, '.format(total_loss),
end="")
print_results(scores, scores_class, self.show_class, t2 - t1)
# print("不同类别：")
# t = classification_report(truths, preds,target_names=["NA","父母子女", "祖孙", "兄弟姐妹", "叔伯姑舅姨", "夫妻", "其他亲戚", "好友", "上下级", "师生", "合作", "情侣", "对立", "共现", "同学", "同门"])
# print(t)
if save_predictions:
test_result = test_result[:w + 1]
test_result_pred = []
test_result_info = []
for item in test_result:
test_result_pred.append([(item[-3], item[1])]) #预测的关系是的概率
test_result_info.append([test_info[item[-2]][item[-1]]])
assert (
item[-3]
in test_info[item[-2]][item[-1]]['rel']) == item[0], print(
"item\n", item, "\n", test_info[item[-2]][item[-1]])
write_errors(test_result_pred,
test_result_info,
self.preds_file,
map_=self.loader.index2rel,
type="theta")
write_preds(test_result_pred,
test_result_info,
self.preds_file,
map_=self.loader.index2rel)
# f1_score_t=f1_score(truths, preds, average='micro')
# print(f1, scores['micro_f'], f1_score_t)
return f1, scores['micro_f'], input_theta, p, r
def train_epoch(self, epoch):
"""
Evaluate the model on the train set.
"""
t1 = time()
output = {
'tp': [],
'fp': [],
'fn': [],
'tn': [],
'loss': [],
'preds': [],
'true': 0,
'ttotal': []
}
self.acc_NA.clear()
self.acc_not_NA.clear()
self.acc_total.clear()
self.model.train()
# 多个batch的数据
train_iter = self.iterator(self.data['train'],
batch_size=self.params['batch'],
shuffle_=self.params['shuffle_data'])
for batch_idx, batch in enumerate(train_iter):
# print("batch_idx", batch_idx)
batch = self.convert_batch(batch, istrain=True)
# with autograd.detect_anomaly():
self.optimizer.zero_grad()
loss, stats, predictions, select, pred_pairs, multi_truths, mask, relation_label = self.model(
batch)
pred_pairs = torch.sigmoid(pred_pairs)
# self.optimizer.zero_grad()
loss.backward() # backward computation
nn.utils.clip_grad_norm_(self.model.parameters(),
self.gc) # gradient clipping
self.optimizer.step() # update
if self.scheduler != None:
self.scheduler.step()
relation_label = relation_label.to('cpu').data.numpy()
predictions = predictions.to('cpu').data.numpy()
# batches
output['loss'] += [float(loss.item())]
output['tp'] += [stats['tp'].to('cpu').data.numpy()]
output['fp'] += [stats['fp'].to('cpu').data.numpy()]
output['fn'] += [stats['fn'].to('cpu').data.numpy()]
output['tn'] += [stats['tn'].to('cpu').data.numpy()]
output['preds'] += [predictions]
output['ttotal'] += [stats['ttotal']]
for i in range(predictions.shape[0]):
label = relation_label[i]
if label < 0:
break
if label == self.loader.label2ignore:
self.acc_NA.add(predictions[i] == label)
else:
self.acc_not_NA.add(predictions[i] == label)
self.acc_total.add(predictions[i] == label)
# 一个epoch
total_loss, scores = self.performance(output)
t2 = time()
self.train_res['loss'] += [total_loss]
self.train_res['score'] += [scores[self.primary_metric]]
self.train_res['p'] += [scores['micro_p']]
self.train_res['r'] += [scores['micro_r']]
print(
'Epoch: {:02d} | TRAIN | LOSS = {:.05f}, | NA acc: {:4.2f} | not NA acc: {:4.2f} | tot acc: {:4.2f}'
.format(epoch, total_loss, self.acc_NA.get(),
self.acc_not_NA.get(), self.acc_total.get()),
end="")
print_results(scores, [], False, t2 - t1)
print("TTotal\t", sum(output['ttotal']))
return scores['micro_f']
def eval_epoch(self, final=False, save_predictions=False):
"""
Evaluate the model on the test set.
No backward computation is allowed.
"""
t1 = time()
output = {
'tp': [],
'fp': [],
'fn': [],
'tn': [],
'loss': [],
'preds': [],
'true': 0
}
test_info = []
test_result = []
self.model.eval()
test_iter = self.iterator(self.data['test'],
batch_size=self.params['batch'],
shuffle_=False)
for batch_idx, batch in enumerate(test_iter):
batch = self.convert_batch(batch, istrain=False, save=True)
with torch.no_grad():
loss, stats, predictions, select, pred_pairs, multi_truths, mask, _ = self.model(
batch) # pred_pairs <#pair, relations_num>
pred_pairs = torch.sigmoid(pred_pairs)
output['loss'] += [loss.item()]
output['tp'] += [stats['tp'].to('cpu').data.numpy()]
output['fp'] += [stats['fp'].to('cpu').data.numpy()]
output['fn'] += [stats['fn'].to('cpu').data.numpy()]
output['tn'] += [stats['tn'].to('cpu').data.numpy()]
output['preds'] += [predictions.to('cpu').data.numpy()]
if True:
test_infos = batch['info'][
select[0].to('cpu').data.numpy(),
select[1].to('cpu').data.numpy(),
select[2].to('cpu').data.numpy()][mask.to(
'cpu').data.numpy()]
test_info += [test_infos]
pred_pairs = pred_pairs.data.cpu().numpy()
multi_truths = multi_truths.data.cpu().numpy()
output['true'] += multi_truths.sum(
) - multi_truths[:, self.loader.label2ignore].sum()
if save_predictions:
assert test_infos.shape[0] == len(pred_pairs), print(
"test info=%d, pred_pair=%d" %
(len(test_infos.shape[0]), len(pred_pairs)))
for pair_id in range(len(pred_pairs)):
multi_truth = multi_truths[pair_id]
for r in range(0, self.rel_size):
if r == self.loader.label2ignore:
continue
test_result.append((int(multi_truth[r]) == 1,
float(pred_pairs[pair_id][r]),
test_infos[pair_id]['intrain'],
test_infos[pair_id]['cross'],
self.loader.index2rel[r], r,
len(test_info) - 1, pair_id))
# estimate performance
total_loss, scores = self.performance(output)
test_result.sort(key=lambda x: x[1], reverse=True)
input_theta, w, f1 = self.tune_f1_theta(test_result,
output['true'],
self.params['input_theta'],
isTest=save_predictions)
t2 = time()
if not final:
self.test_res['loss'] += [total_loss]
# self.test_res['score'] += [scores[self.primary_metric]]
self.test_res['score'] += [f1]
print(' TEST | LOSS = {:.05f}, '.format(total_loss),
end="")
print_results(scores, [], self.show_class, t2 - t1)
print()
if save_predictions:
test_result = test_result[:w + 1]
test_result_pred = []
test_result_info = []
for item in test_result:
test_result_pred.append([(item[-3], item[1])])
test_result_info.append([test_info[item[-2]][item[-1]]])
assert (
item[-3]
in test_info[item[-2]][item[-1]]['rel']) == item[0], print(
"item\n", item, "\n", test_info[item[-2]][item[-1]])
write_errors(test_result_pred,
test_result_info,
self.preds_file,
map_=self.loader.index2rel,
type="theta")
write_preds(test_result_pred,
test_result_info,
self.preds_file,
map_=self.loader.index2rel)
return f1, scores['micro_f'], input_theta