def run_test(args):
from pybert.test.predictor import Predictor
processor = BertProcessor(vocab_path=config['bert_vocab_path'],
do_lower_case=args.do_lower_case)
test_data = processor.get_test(config['test_path'])
test_examples = processor.create_examples(
lines=test_data,
example_type='test',
cached_examples_file=config['data_dir'] /
f"cached_test_examples_{args.arch}")
test_features = processor.create_features(
examples=test_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_test_features_{}_{}".format(args.eval_max_seq_len, args.arch))
test_dataset = processor.create_dataset(test_features)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset,
sampler=test_sampler,
batch_size=args.eval_batch_size)
idx2word = {}
for (w, i) in processor.tokenizer.vocab.items():
idx2word[i] = w
label_list = processor.get_labels(label_path=config['data_label_path'])
idx2label = {i: label for i, label in enumerate(label_list)}
if args.test_path:
args.test_path = Path(args.test_path)
model = BertForMultiLable.from_pretrained(args.test_path,
num_labels=len(label_list))
else:
model = BertForMultiLable.from_pretrained(config['bert_model_dir'],
num_labels=len(label_list))
for p in model.bert.parameters():
p.require_grad = False
# ----------- predicting -----------
writer = SummaryWriter()
logger.info('model predicting....')
predictor = Predictor(model=model,
logger=logger,
n_gpu=args.n_gpu,
i2w=idx2word,
i2l=idx2label)
result = predictor.predict(data=test_dataloader)
if args.predict_labels:
predictor.labels(result, args.predict_idx)
def run_test(args):
from pybert.io.task_data import TaskData
from pybert.test.predictor import Predictor
data = TaskData()
targets, sentences = data.read_data(raw_data_path=config['test_path'],
preprocessor=EnglishPreProcessor(),
is_train=False)
lines = list(zip(sentences, targets))
processor = BertProcessor(vocab_path=config['bert_vocab_path'], do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
id2label = {i: label for i, label in enumerate(label_list)}
test_data = processor.get_test(lines=lines)
test_examples = processor.create_examples(lines=test_data,
example_type='test',
cached_examples_file=config[
'data_dir'] / f"cached_test_examples_{args.arch}")
test_features = processor.create_features(examples=test_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config[
'data_dir'] / "cached_test_features_{}_{}".format(
args.eval_max_seq_len, args.arch
))
test_dataset = processor.create_dataset(test_features)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=args.train_batch_size)
model = BertForMultiLable.from_pretrained(config['checkpoint_dir'], num_labels=len(label_list))
# ----------- predicting
logger.info('model predicting....')
predictor = Predictor(model=model,
logger=logger,
n_gpu=args.n_gpu)
result = predictor.predict(data=test_dataloader)
print(result)
def run_test(args):
from pybert.io.task_data import TaskData
from pybert.test.predictor import Predictor
import pickle
import os
processor = BertProcessor(vocab_path=config['bert_vocab_path'], do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
label2id = {label: i for i, label in enumerate(label_list)}
id2label = {i: label for i, label in enumerate(label_list)}
test_data = processor.get_train(config['data_dir'] / f"{args.data_name}.test.pkl")
print ("Test data is:")
print (test_data)
print ("Label list is:")
print (label_list)
print ("----------------------------------------")
# test_data = processor.get_test(lines=lines)
test_examples = processor.create_examples(lines=test_data,
example_type='test',
cached_examples_file=config[
'data_cache'] / f"cached_test_examples_{args.arch}")
test_features = processor.create_features(examples=test_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config[
'data_cache'] / "cached_test_features_{}_{}".format(
args.eval_max_seq_len, args.arch
))
test_dataset = processor.create_dataset(test_features)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=args.train_batch_size)
model = BertForMultiLable.from_pretrained(config['checkpoint_dir'], num_labels=len(label_list))
# ----------- predicting
logger.info('model predicting....')
predictor = Predictor(model=model,
logger=logger,
n_gpu=args.n_gpu,
batch_metrics=[AccuracyThresh(thresh=0.5)],
epoch_metrics=[AUC(average='micro', task_type='binary'),
MultiLabelReport(id2label=id2label)])
result, test_predicted, test_true = predictor.predict(data=test_dataloader)
pickle.dump(test_true, open(os.path.join(config["test/checkpoint_dir"], "test_true.p"), "wb"))
pickle.dump(test_predicted, open(os.path.join(config["test/checkpoint_dir"], "test_predicted.p"), "wb"))
pickle.dump(id2label, open(os.path.join(config["test/checkpoint_dir"], "id2label.p"), "wb"))
print ("Predictor results:")
print(result)
print ("-----------------------------------------------")
def run_train(args):
# --------- data
processor = BertProcessor(vocab_path=config['bert_vocab_path'],
do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
label2id = {label: i for i, label in enumerate(label_list)}
id2label = {i: label for i, label in enumerate(label_list)}
train_data = processor.get_train(config['data_dir'] /
f"{args.data_name}.label_train.pkl")
print("Train data is:")
print(train_data)
train_examples = processor.create_examples(
lines=train_data,
example_type='train',
cached_examples_file=config['data_cache'] /
f"cached_train_label_examples_finetune{args.arch}")
# print ("Training examples are:")
# print (train_examples)
train_features = processor.create_features(
examples=train_examples,
max_seq_len=args.train_max_seq_len,
cached_features_file=config['data_cache'] /
"cached_train_label_features_finetune{}_{}".format(
args.train_max_seq_len, args.arch))
train_dataset = processor.create_dataset(train_features,
is_sorted=args.sorted)
if args.sorted:
train_sampler = SequentialSampler(train_dataset)
else:
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
valid_data = processor.get_dev(config['data_dir'] /
f"{args.data_name}.label_valid.pkl")
valid_examples = processor.create_examples(
lines=valid_data,
example_type='valid',
cached_examples_file=config['data_cache'] /
f"cached_valid_examples_label_finetune{args.arch}")
valid_features = processor.create_features(
examples=valid_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_cache'] /
"cached_valid_features_label_finetune{}_{}".format(
args.eval_max_seq_len, args.arch))
valid_dataset = processor.create_dataset(valid_features)
valid_sampler = SequentialSampler(valid_dataset)
valid_dataloader = DataLoader(valid_dataset,
sampler=valid_sampler,
batch_size=args.eval_batch_size)
# ------- model
logger.info("initializing model")
if args.resume_path:
args.resume_path = Path(args.resume_path)
model = BertForMultiLable.from_pretrained(args.resume_path,
num_labels=len(label_list))
else:
print("Labels are:")
print(label_list)
# model = BertForMultiLable.from_pretrained(config['bert_model_dir'], num_labels=len(label_list))
model = BertForMultiLable.from_pretrained("bert-base-uncased",
num_labels=len(label_list))
t_total = int(
len(train_dataloader) / args.gradient_accumulation_steps * args.epochs)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
args.weight_decay
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
warmup_steps = int(t_total * args.warmup_proportion)
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
lr_scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=warmup_steps,
t_total=t_total)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
# ---- callbacks
logger.info("initializing callbacks")
train_monitor = TrainingMonitor(file_dir=config['figure_dir'],
arch=args.arch)
model_checkpoint = ModelCheckpoint(checkpoint_dir=config['checkpoint_dir'],
mode=args.mode,
monitor=args.monitor,
arch=args.arch,
save_best_only=args.save_best)
# **************************** training model ***********************
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Num Epochs = %d", args.epochs)
logger.info(
" Total train batch size (w. parallel, distributed & accumulation) = %d",
args.train_batch_size * args.gradient_accumulation_steps *
(torch.distributed.get_world_size() if args.local_rank != -1 else 1))
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Total optimization steps = %d", t_total)
trainer = Trainer(
n_gpu=args.n_gpu,
model=model,
epochs=args.epochs,
logger=logger,
criterion=BCEWithLogLoss(),
optimizer=optimizer,
lr_scheduler=lr_scheduler,
early_stopping=None,
training_monitor=train_monitor,
fp16=args.fp16,
resume_path=args.resume_path,
grad_clip=args.grad_clip,
model_checkpoint=model_checkpoint,
gradient_accumulation_steps=args.gradient_accumulation_steps,
batch_metrics=[AccuracyThresh(thresh=0.5)],
epoch_metrics=[
AUC(average='micro', task_type='binary'),
MultiLabelReport(id2label=id2label)
])
# embeddings_dict = pickle.load(open("/home/rgaonkar/context_home/rgaonkar/label_embeddings/code/Bert_Masked_LM/label_embeddings_dict.p", "rb"))
# label_similarity_matrix = get_label_similarity_matrix(embeddings_dict, label_list)
trainer.train(train_data=train_dataloader,
valid_data=valid_dataloader,
seed=args.seed)
def run_test(args, test=False, k=7, med_map='pybert/dataset/med_map.csv'):
from pybert.io.task_data import TaskData
from pybert.test.predictor import Predictor
data = TaskData()
targets, sentences = data.read_data(raw_data_path=config['test_path'],
preprocessor=EnglishPreProcessor(),
is_train=test)
print(
f'-----------------------------------------\ntargets {targets}\n---------------------------------------------------'
)
lines = list(zip(sentences, targets))
processor = BertProcessor(vocab_path=config['bert_vocab_path'],
do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
id2label = {i: label for i, label in enumerate(label_list)}
test_data = processor.get_test(lines=lines)
test_examples = processor.create_examples(
lines=test_data,
example_type='test',
cached_examples_file=config['data_dir'] /
f"cached_test_examples_{args.arch}")
test_features = processor.create_features(
examples=test_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_test_features_{}_{}".format(args.eval_max_seq_len, args.arch))
test_dataset = processor.create_dataset(test_features)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset,
sampler=test_sampler,
batch_size=args.train_batch_size)
model = BertForMultiLable.from_pretrained(config['checkpoint_dir'])
# ----------- predicting
logger.info('model predicting....')
predictor = Predictor(model=model,
logger=logger,
n_gpu=args.n_gpu,
test=test)
if test:
results, targets = predictor.predict(data=test_dataloader)
#print(f'results {results.shape}')
#print(f'targets {targets.shape}')
result = dict()
metrics = [Recall(), Acc()]
for metric in metrics:
metric.reset()
metric(logits=results, target=targets)
value = metric.value()
if value is not None:
result[f'valid_{metric.name()}'] = value
return result
else:
results = predictor.predict(data=test_dataloader)
pred = np.argsort(results)[:, -k:][:, ::-1]
with open('pybert/dataset/med_map.csv', mode='r') as infile:
reader = csv.reader(infile)
med_dict = {int(rows[0]): rows[1] for rows in reader}
pred = np.vectorize(med_dict.get)(pred)
return pred
def run_train(args):
# --------- data ---------
processor = BertProcessor(vocab_path=config['bert_vocab_path'],
do_lower_case=args.do_lower_case)
idx2word = {}
for (w, i) in processor.tokenizer.vocab.items():
idx2word[i] = w
label_list = processor.get_labels(label_path=config['data_label_path'])
idx2label = {i: label for i, label in enumerate(label_list)}
train_data = processor.get_train(config['data_dir'] /
f"{args.data_name}.train.pkl")
train_examples = processor.create_examples(
lines=train_data,
example_type='train',
cached_examples_file=config['data_dir'] /
f"cached_train_examples_{args.arch}")
train_features = processor.create_features(
examples=train_examples,
max_seq_len=args.train_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_train_features_{}_{}".format(args.train_max_seq_len,
args.arch))
train_dataset = processor.create_dataset(train_features,
is_sorted=args.sorted)
if args.sorted:
train_sampler = SequentialSampler(train_dataset)
else:
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
valid_data = processor.get_dev(config['data_dir'] /
f"{args.data_name}.valid.pkl")
valid_examples = processor.create_examples(
lines=valid_data,
example_type='valid',
cached_examples_file=config['data_dir'] /
f"cached_valid_examples_{args.arch}")
valid_features = processor.create_features(
examples=valid_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_valid_features_{}_{}".format(args.eval_max_seq_len, args.arch))
valid_dataset = processor.create_dataset(valid_features)
valid_sampler = SequentialSampler(valid_dataset)
valid_dataloader = DataLoader(valid_dataset,
sampler=valid_sampler,
batch_size=args.eval_batch_size)
# ------- model -------
logger.info("initializing model")
if args.resume_path:
args.resume_path = Path(args.resume_path)
model = BertForMultiLable.from_pretrained(args.resume_path,
num_labels=len(label_list))
else:
model = BertForMultiLable.from_pretrained(config['bert_model_dir'],
num_labels=len(label_list))
for p in model.parameters():
p.requires_grad = False
# training last 2 fc layers
model.classifier.weight.requires_grad = True
model.classifier_1.weight.requires_grad = True
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
args.weight_decay
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
t_total = int(
len(train_dataloader) / args.gradient_accumulation_steps * args.epochs)
warmup_steps = int(t_total * args.warmup_proportion)
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
lr_scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=warmup_steps,
t_total=t_total)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
# ---- callbacks ----
logger.info("initializing callbacks")
train_monitor = TrainingMonitor(file_dir=config['figure_dir'],
arch=args.arch)
model_checkpoint = ModelCheckpoint(checkpoint_dir=config['checkpoint_dir'],
mode=args.mode,
monitor=args.monitor,
arch=args.arch,
save_best_only=args.save_best)
# **************************** training model ***********************
writer = SummaryWriter()
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Num Epochs = %d", args.epochs)
logger.info(
" Total train batch size (w. parallel, distributed & accumulation) = %d",
args.train_batch_size * args.gradient_accumulation_steps *
(torch.distributed.get_world_size() if args.local_rank != -1 else 1))
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Total optimization steps = %d", t_total)
trainer = Trainer(
n_gpu=args.n_gpu,
i2w=idx2word,
i2l=idx2label,
model=model,
epochs=args.epochs,
logger=logger,
criterion=BCEWithLogLoss(),
optimizer=optimizer,
lr_scheduler=lr_scheduler,
early_stopping=None,
training_monitor=train_monitor,
fp16=args.fp16,
resume_path=args.resume_path,
grad_clip=args.grad_clip,
model_checkpoint=model_checkpoint,
gradient_accumulation_steps=args.gradient_accumulation_steps,
batch_metrics=[AccuracyThresh(thresh=0.5)],
epoch_metrics=[],
writer=writer,
)
trainer.train(train_data=train_dataloader,
valid_data=valid_dataloader,
seed=args.seed)