def evaluate(args):
logger = logging.getLogger('Causality')
logger.info('Loading valid file...')
with open(args.valid_record_file, 'rb') as fh:
valid_file = pkl.load(fh)
fh.close()
logger.info('Loading test file...')
with open(args.test_record_file, 'rb') as fh:
test_file = pkl.load(fh)
fh.close()
logger.info('Loading valid meta...')
valid_meta = load_json(args.valid_meta)
logger.info('Loading test meta...')
test_meta = load_json(args.test_meta)
logger.info('Loading id to token file...')
id2token_file = load_json(args.id2token_file)
logger.info('Loading token embeddings...')
with open(args.token_emb_file, 'rb') as fh:
token_embeddings = pkl.load(fh)
fh.close()
valid_num = valid_meta['total']
test_num = test_meta['total']
logger.info('Loading shape meta...')
logger.info('Num valid data {} test data {}'.format(valid_num, test_num))
args.dropout = {'emb': args.emb_dropout, 'layer': args.layer_dropout}
model = getattr(models, args.model)(token_embeddings, args,
logger).to(device=args.device)
model.load_state_dict(torch.load(os.path.join(args.model_dir,
'model.bin')))
eval_metrics, fpr, tpr, precision, recall = evaluate_batch(
model, test_num, args.batch_eval, test_file, args.device, args.is_fc,
'eval', logger)
logger.info('Eval Loss - {}'.format(eval_metrics['loss']))
logger.info('Eval Acc - {}'.format(eval_metrics['acc']))
logger.info('Eval Precision - {}'.format(eval_metrics['precision']))
logger.info('Eval Recall - {}'.format(eval_metrics['recall']))
logger.info('Eval F1 - {}'.format(eval_metrics['f1']))
logger.info('Eval AUCROC - {}'.format(eval_metrics['auc_roc']))
logger.info('Eval AUCPRC - {}'.format(eval_metrics['auc_prc']))
# if args.model == 'MCDN' or args.model == 'TB':
# draw_att(model, test_num, args.batch_eval, test_file, args.device, id2token_file,
# args.pics_dir, args.n_block, args.n_head, logger)
FALSE = {'FP': eval_metrics['fp'], 'FN': eval_metrics['fn']}
ROC = {'FPR': fpr, 'TPR': tpr}
PRC = {'PRECISION': precision, 'RECALL': recall}
dump_json(os.path.join(args.result_dir, 'FALSE_transfer.json'), FALSE)
dump_json(os.path.join(args.result_dir, 'ROC_transfer.json'), ROC)
dump_json(os.path.join(args.result_dir, 'PRC_transfer.json'), PRC)
draw_curve(ROC['FPR'], ROC['TPR'], PRC['PRECISION'], PRC['RECALL'],
args.pics_dir)
def train(args):
logger = logging.getLogger('Causality')
logger.info('Loading train file...')
with open(args.train_record_file, 'rb') as fh:
train_file = pkl.load(fh)
fh.close()
logger.info('Loading valid file...')
with open(args.valid_record_file, 'rb') as fh:
valid_file = pkl.load(fh)
fh.close()
logger.info('Loading train meta...')
train_meta = load_json(args.train_meta)
logger.info('Loading valid meta...')
valid_meta = load_json(args.valid_meta)
logger.info('Loading token embeddings...')
with open(args.token_emb_file, 'rb') as fh:
token_embeddings = pkl.load(fh)
fh.close()
train_num = train_meta['total']
valid_num = valid_meta['total']
logger.info('Loading shape meta...')
logger.info('Num train data {} valid data {}'.format(train_num, valid_num))
args.dropout = {'emb': args.emb_dropout, 'layer': args.layer_dropout}
logger.info('Initialize the model...')
model = getattr(models, args.model)(token_embeddings, args,
logger).to(device=args.device)
lr = args.lr
optimizer = getattr(optim, args.optim)(model.parameters(),
lr=lr,
weight_decay=args.weight_decay)
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', 0.5, patience=args.patience, verbose=True)
scheduler = WarmupCosineSchedule(optimizer, args.warmup,
(train_num // args.batch_train + 1) *
args.epochs)
# torch.backends.cudnn.benchmark = True
max_acc, max_p, max_r, max_f, max_roc, max_prc, max_sum, max_epoch = np.zeros(
8)
FALSE, ROC, PRC = {}, {}, {}
train_loss, valid_loss = [], []
for ep in range(1, args.epochs + 1):
logger.info('Training the model for epoch {}'.format(ep))
avg_loss = train_one_epoch(model, optimizer, scheduler, train_num,
train_file, args, logger)
train_loss.append(avg_loss)
logger.info('Epoch {} AvgLoss {}'.format(ep, avg_loss))
logger.info('Evaluating the model for epoch {}'.format(ep))
eval_metrics, fpr, tpr, precision, recall = evaluate_batch(
model, valid_num, args.batch_eval, valid_file, args.device,
args.is_fc, 'valid', logger)
valid_loss.append(eval_metrics['loss'])
logger.info('Valid Loss - {}'.format(eval_metrics['loss']))
logger.info('Valid Acc - {}'.format(eval_metrics['acc']))
logger.info('Valid Precision - {}'.format(eval_metrics['precision']))
logger.info('Valid Recall - {}'.format(eval_metrics['recall']))
logger.info('Valid F1 - {}'.format(eval_metrics['f1']))
logger.info('Valid AUCROC - {}'.format(eval_metrics['auc_roc']))
logger.info('Valid AUCPRC - {}'.format(eval_metrics['auc_prc']))
max_acc = max((eval_metrics['acc'], max_acc))
max_p = max(eval_metrics['precision'], max_p)
max_r = max(eval_metrics['recall'], max_r)
max_f = max(eval_metrics['f1'], max_f)
valid_sum = eval_metrics['auc_roc'] + eval_metrics[
'auc_prc'] + eval_metrics['f1']
if valid_sum > max_sum:
max_acc = eval_metrics['acc']
max_p = eval_metrics['precision']
max_r = eval_metrics['recall']
max_f = eval_metrics['f1']
max_roc = eval_metrics['auc_roc']
max_prc = eval_metrics['auc_prc']
max_sum = valid_sum
max_epoch = ep
FALSE = {'FP': eval_metrics['fp'], 'FN': eval_metrics['fn']}
ROC = {'FPR': fpr, 'TPR': tpr}
PRC = {'PRECISION': precision, 'RECALL': recall}
torch.save(model.state_dict(),
os.path.join(args.model_dir, 'model.bin'))
# scheduler.step(metrics=eval_metrics['f1'])
random.shuffle(train_file)
logger.info('Max Acc - {}'.format(max_acc))
logger.info('Max Precision - {}'.format(max_p))
logger.info('Max Recall - {}'.format(max_r))
logger.info('Max F1 - {}'.format(max_f))
logger.info('Max ROC - {}'.format(max_roc))
logger.info('Max PRC - {}'.format(max_prc))
logger.info('Max Epoch - {}'.format(max_epoch))
logger.info('Max Sum - {}'.format(max_sum))
dump_json(os.path.join(args.result_dir, 'FALSE_valid.json'), FALSE)
dump_json(os.path.join(args.result_dir, 'ROC_valid.json'), ROC)
dump_json(os.path.join(args.result_dir, 'PRC_valid.json'), PRC)
save_loss(train_loss, valid_loss, args.result_dir)
draw_curve(ROC['FPR'], ROC['TPR'], PRC['PRECISION'], PRC['RECALL'],
args.pics_dir)
def train(args, file_paths):
logger = logging.getLogger('2010')
logger.info('Loading train file...')
with open(file_paths.train_record_file, 'rb') as fh:
train_file = pkl.load(fh)
fh.close()
logger.info('Loading valid file...')
with open(file_paths.valid_record_file, 'rb') as fh:
valid_file = pkl.load(fh)
fh.close()
logger.info('Loading train meta...')
with open(file_paths.train_meta, 'r') as fh:
train_meta = json.load(fh)
fh.close()
logger.info('Loading valid meta...')
with open(file_paths.valid_meta, 'r') as fh:
valid_meta = json.load(fh)
fh.close()
logger.info('Loading token embeddings...')
with open(file_paths.token_emb_file, 'rb') as fh:
token_embeddings = pkl.load(fh)
fh.close()
train_num = train_meta['total']
valid_num = valid_meta['total']
logger.info('Loading shape meta...')
logger.info('Num train data {} valid data {}'.format(train_num, valid_num))
dropout = {'emb': args.emb_dropout, 'layer': args.layer_dropout}
logger.info('Initialize the model...')
model = MCIN(token_embeddings, args.max_len, args.n_class, args.n_hidden,
args.n_layer, args.n_kernels, args.n_filter, args.n_block,
args.n_head, args.is_sinusoid, args.is_ffn, dropout,
logger).to(device=args.device)
# model = SCRN(token_embeddings, args.max_len, args.n_class, args.n_hidden, args.n_layer,
# args.n_kernels, args.n_filter, args.n_block, args.n_head, args.is_sinusoid, args.is_ffn,
# dropout, logger).to(device=args.device)
# model = TB(token_embeddings, args.max_len, args.n_class, args.n_hidden, args.n_layer,
# args.n_kernels, args.n_filter, args.n_block, args.n_head, args.is_sinusoid, args.is_ffn,
# dropout, logger).to(device=args.device)
lr = args.lr
optimizer = getattr(optim, args.optim)(model.parameters(),
lr=lr,
weight_decay=args.weight_decay)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
'max',
0.5,
patience=args.patience,
verbose=True)
max_acc, max_p, max_r, max_f, max_sum, max_epoch = 0, 0, 0, 0, 0, 0
FALSE = {}
ROC = {}
PRC = {}
for ep in range(1, args.epochs + 1):
logger.info('Training the model for epoch {}'.format(ep))
avg_loss = train_one_epoch(model, optimizer, train_num, train_file,
args, logger)
logger.info('Epoch {} AvgLoss {}'.format(ep, avg_loss))
logger.info('Evaluating the model for epoch {}'.format(ep))
eval_metrics, fpr, tpr, precision, recall = evaluate_batch(
model, valid_num, args.batch_eval, valid_file, args.device,
args.is_fc, 'valid', logger)
logger.info('Valid Loss - {}'.format(eval_metrics['loss']))
logger.info('Valid Acc - {}'.format(eval_metrics['acc']))
logger.info('Valid Precision - {}'.format(eval_metrics['precision']))
logger.info('Valid Recall - {}'.format(eval_metrics['recall']))
logger.info('Valid F1 - {}'.format(eval_metrics['f1']))
logger.info('Valid AUCROC - {}'.format(eval_metrics['auc_roc']))
logger.info('Valid AUCPRC - {}'.format(eval_metrics['auc_prc']))
max_acc = max((eval_metrics['acc'], max_acc))
max_p = max(eval_metrics['precision'], max_p)
max_r = max(eval_metrics['recall'], max_r)
max_f = max(eval_metrics['f1'], max_f)
valid_sum = eval_metrics['precision'] + eval_metrics[
'recall'] + eval_metrics['f1']
if valid_sum > max_sum:
max_sum = valid_sum
max_epoch = ep
FALSE = {'FP': eval_metrics['fp'], 'FN': eval_metrics['fn']}
ROC = {'FPR': fpr, 'TPR': tpr}
PRC = {'PRECISION': precision, 'RECALL': recall}
# torch.save(model, os.path.join(args.model_dir, 'model.pth'))
torch.save(model.state_dict(),
os.path.join(args.model_dir, 'model.bin'))
scheduler.step(metrics=eval_metrics['f1'])
random.shuffle(train_file)
logger.info('Max Acc - {}'.format(max_acc))
logger.info('Max Precision - {}'.format(max_p))
logger.info('Max Recall - {}'.format(max_r))
logger.info('Max F1 - {}'.format(max_f))
logger.info('Max Epoch - {}'.format(max_epoch))
logger.info('Max Sum - {}'.format(max_sum))
with open(os.path.join(args.result_dir, 'FALSE.json'), 'w') as f:
f.write(json.dumps(FALSE) + '\n')
f.close()
with open(os.path.join(args.result_dir, 'ROC.json'), 'w') as f:
f.write(json.dumps(ROC) + '\n')
f.close()
with open(os.path.join(args.result_dir, 'PRC.json'), 'w') as f:
f.write(json.dumps(PRC) + '\n')
f.close()
draw_curve(ROC['FPR'], ROC['TPR'], PRC['PRECISION'], PRC['RECALL'],
args.pics_dir)
def train(args, file_paths):
logger = logging.getLogger('Causality')
logger.info('Loading train file...')
with open(file_paths.train_record_file, 'rb') as fh:
train_file = pkl.load(fh)
fh.close()
logger.info('Loading valid file...')
with open(file_paths.valid_record_file, 'rb') as fh:
valid_file = pkl.load(fh)
fh.close()
logger.info('Loading train meta...')
train_meta = load_json(file_paths.train_meta)
logger.info('Loading valid meta...')
valid_meta = load_json(file_paths.valid_meta)
logger.info('Loading token embeddings...')
with open(file_paths.token_emb_file, 'rb') as fh:
token_embeddings = pkl.load(fh)
fh.close()
train_num = train_meta['total']
valid_num = valid_meta['total']
logger.info('Loading shape meta...')
logger.info('Num train data {} valid data {}'.format(train_num, valid_num))
args.dropout = {'emb': args.emb_dropout, 'layer': args.layer_dropout}
records = np.zeros((args.multi, 7), dtype=np.float)
best_sum = -1
for i in range(1, args.multi + 1):
logger.info('Initialize the model...')
# model = getattr(models, args.model)(token_embeddings, args, logger).to(device=args.device)
# model = TCN(token_embeddings, args.max_len['full'], args.n_class, n_channel=[args.n_filter] * args.n_level,
# n_kernel=args.n_kernel, n_block=args.n_block, n_head=args.n_head, dropout=dropout, logger=logger).
# to(device=args.device)
# model = BiGRU(token_embeddings, args.max_len['full'], args.n_class, args.n_hidden, args.n_layer, args.n_block,
# args.n_head, args.is_sinusoid, args.is_ffn, dropout, logger).to(device=args.device)
# model = TextCNN(token_embeddings, args.max_len, args.n_class, args.n_kernels, args.n_filter, args.is_pos,
# args.is_sinusoid, args.dropout, logger).to(device=args.device)
# model = TextCNNDeep(token_embeddings, args.max_len, args.n_class, args.n_kernels, args.n_filter,
# args.dropout, logger).to(device=args.device)
# model = DPCNN(token_embeddings, args, logger).to(device=args.device)
# model = TextRNN(token_embeddings, args.n_class, args.n_hidden, args.n_layer, args.kmax_pooling,
# args.is_pos, args.is_sinusoid, args.dropout, logger).to(device=args.device)
model = SelfAttentive(token_embeddings, args.n_class, args.n_hidden,
args.n_layer, 128, 32, args.dropout,
logger).to(device=args.device)
lr = args.lr
optimizer = getattr(optim, args.optim)(model.parameters(),
lr=lr,
weight_decay=args.weight_decay)
# scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', 0.5, patience=args.patience, verbose=True)
scheduler = WarmupCosineSchedule(optimizer, args.warmup,
(train_num // args.batch_train + 1) *
args.epochs)
logger.info('Turn {}'.format(i))
max_acc, max_p, max_r, max_f, max_roc, max_prc, max_sum, max_epoch = np.zeros(
8)
FALSE, ROC, PRC = {}, {}, {}
train_loss, valid_loss = [], []
is_best = False
for ep in range(1, args.epochs + 1):
logger.info('Training the model for epoch {}'.format(ep))
avg_loss = train_one_epoch(model, optimizer, scheduler, train_num,
train_file, args, logger)
train_loss.append(avg_loss)
logger.info('Epoch {} AvgLoss {}'.format(ep, avg_loss))
logger.info('Evaluating the model for epoch {}'.format(ep))
eval_metrics, fpr, tpr, precision, recall = evaluate_batch(
model, valid_num, args.batch_eval, valid_file, args.device,
args.is_fc, 'valid', logger)
valid_loss.append(eval_metrics['loss'])
logger.info('Valid Loss - {}'.format(eval_metrics['loss']))
logger.info('Valid Acc - {}'.format(eval_metrics['acc']))
logger.info('Valid Precision - {}'.format(
eval_metrics['precision']))
logger.info('Valid Recall - {}'.format(eval_metrics['recall']))
logger.info('Valid F1 - {}'.format(eval_metrics['f1']))
logger.info('Valid AUCROC - {}'.format(eval_metrics['auc_roc']))
logger.info('Valid AUCPRC - {}'.format(eval_metrics['auc_prc']))
valid_sum = eval_metrics['acc'] + eval_metrics[
'precision'] + eval_metrics['recall'] + eval_metrics['f1']
if valid_sum > max_sum:
max_acc = eval_metrics['acc']
max_p = eval_metrics['precision']
max_r = eval_metrics['recall']
max_f = eval_metrics['f1']
max_roc = eval_metrics['auc_roc']
max_prc = eval_metrics['auc_prc']
max_sum = valid_sum
max_epoch = ep
FALSE = {'FP': eval_metrics['fp'], 'FN': eval_metrics['fn']}
ROC = {'FPR': fpr, 'TPR': tpr}
PRC = {'PRECISION': precision, 'RECALL': recall}
# torch.save(model, os.path.join(args.model_dir, 'model.pth'))
if max_sum > best_sum:
best_sum = max_sum
is_best = True
torch.save(model.state_dict(),
os.path.join(args.model_dir, 'model.bin'))
random.shuffle(train_file)
logger.info('Max Acc - {}'.format(max_acc))
logger.info('Max Precision - {}'.format(max_p))
logger.info('Max Recall - {}'.format(max_r))
logger.info('Max F1 - {}'.format(max_f))
logger.info('Max ROC - {}'.format(max_roc))
logger.info('Max PRC - {}'.format(max_prc))
logger.info('Max Epoch - {}'.format(max_epoch))
logger.info('Max Sum - {}'.format(max_sum))
if is_best:
dump_json(os.path.join(args.result_dir, 'FALSE_valid.json'), FALSE)
dump_json(os.path.join(args.result_dir, 'ROC_valid.json'), ROC)
dump_json(os.path.join(args.result_dir, 'PRC_valid.json'), PRC)
save_loss(train_loss, valid_loss, args.result_dir)
draw_curve(ROC['FPR'], ROC['TPR'], PRC['PRECISION'], PRC['RECALL'],
args.pics_dir)
records[i - 1] = [
max_acc, max_p, max_r, max_f, max_roc, max_prc, max_epoch
]
save_metrics(records, args.result_dir)