import matplotlib.pyplot as plt
import tkinter as tk
df = pd.read_csv('/Users/iluzaangirova/Ilyuza/dataset_5.csv', encoding='utf8')
df = df[['topics', 'title']]
maps = pd.factorize(df.topics)[1]
device = 'cpu'
bert_config = BertConfig.from_json_file(
'/Users/iluzaangirova/Ilyuza/bert_config.json')
model = BertForSequenceClassification(bert_config, 5)
model.to(device)
model = torch.nn.DataParallel(model)
model.load_state_dict(
torch.load('/Users/iluzaangirova/Ilyuza/weights5000.pth',
map_location='cpu'))
tokenizer = tokenization.FullTokenizer(
vocab_file='/Users/iluzaangirova/Ilyuza/vocab.txt', do_lower_case=False)
def predict(sentence, tokenizer, max_len):
# print(sentence)
tokens_a = tokenizer.tokenize(sentence)
if len(tokens_a) > max_len - 2:
tokens_a = tokens_a[0:(max_len - 2)]
tokens = []
tokens.append("[CLS]")
for token in tokens_a:
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
choices=["sentihood_single", "sentihood_NLI_M", "sentihood_QA_M", \
"sentihood_NLI_B", "sentihood_QA_B", "semeval_single", \
"semeval_NLI_M", "semeval_QA_M", "semeval_NLI_B", "semeval_QA_B"],
help="The name of the task to train.")
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--init_eval_checkpoint",
default=None,
type=str,
help=
"Initial checkpoint (usually from a pre-trained BERT model + classifier)."
)
parser.add_argument("--do_save_model",
default=False,
action='store_true',
help="Whether to save model.")
parser.add_argument("--eval_test",
default=False,
action='store_true',
help="Whether to run eval on the test set.")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
# prepare dataloaders
processors = {
"sentihood_single": Sentihood_single_Processor,
"sentihood_NLI_M": Sentihood_NLI_M_Processor,
"sentihood_QA_M": Sentihood_QA_M_Processor,
"sentihood_NLI_B": Sentihood_NLI_B_Processor,
"sentihood_QA_B": Sentihood_QA_B_Processor,
"semeval_single": Semeval_single_Processor,
"semeval_NLI_M": Semeval_NLI_M_Processor,
"semeval_QA_M": Semeval_QA_M_Processor,
"semeval_NLI_B": Semeval_NLI_B_Processor,
"semeval_QA_B": Semeval_QA_B_Processor,
}
processor = processors[args.task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
# training set
train_examples = None
num_train_steps = None
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs)
train_features = convert_examples_to_features(train_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
# test set
if args.eval_test:
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
test_dataloader = DataLoader(test_data,
batch_size=args.eval_batch_size,
shuffle=False)
# model and optimizer
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_eval_checkpoint is not None:
model.load_state_dict(
torch.load(args.init_eval_checkpoint, map_location='cpu'))
elif args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.01
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
# train
output_log_file = os.path.join(args.output_dir, "log.txt")
print("output_log_file=", output_log_file)
with open(output_log_file, "w") as writer:
if args.eval_test:
writer.write(
"epoch\tglobal_step\tloss\ttest_loss\ttest_accuracy\n")
else:
writer.write("epoch\tglobal_step\tloss\n")
global_step = 0
epoch = 0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
epoch += 1
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if args.do_save_model:
if n_gpu > 1:
torch.save(
model.module.state_dict(),
os.path.join(args.output_dir, f'model_ep_{epoch}.bin'))
else:
torch.save(
model.state_dict(),
os.path.join(args.output_dir, f'model_ep_{epoch}.bin'))
# eval_test
if args.eval_test:
model.eval()
test_loss, test_accuracy = 0, 0
nb_test_steps, nb_test_examples = 0, 0
with open(os.path.join(args.output_dir, f"test_ep_{epoch}.txt"),
"w") as f_test:
for input_ids, input_mask, segment_ids, label_ids in test_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_test_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids)
logits = F.softmax(logits, dim=-1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
outputs = np.argmax(logits, axis=1)
for output_i in range(len(outputs)):
f_test.write(str(outputs[output_i]))
for ou in logits[output_i]:
f_test.write(" " + str(ou))
f_test.write("\n")
tmp_test_accuracy = np.sum(outputs == label_ids)
test_loss += tmp_test_loss.mean().item()
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
result = collections.OrderedDict()
if args.eval_test:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps,
'test_loss': test_loss,
'test_accuracy': test_accuracy
}
else:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
logger.info("***** Eval results *****")
with open(output_log_file, "a+") as writer:
for key in result.keys():
logger.info(" %s = %s\n", key, str(result[key]))
writer.write("%s\t" % (str(result[key])))
writer.write("\n")
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--train_file",
default=None,
type=str,
required=True,
help="The train file path")
parser.add_argument("--eval_file",
default=None,
type=str,
required=True,
help="The dev file path")
parser.add_argument("--eval_train_file",
default=None,
type=str,
required=True,
help="The train eval file path")
parser.add_argument("--predict_file",
default=None,
type=str,
required=False,
help="The predict file path")
parser.add_argument("--top_n",
default=5,
type=float,
required=True,
help="higher than threshold is classify 1,")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
required=False,
help="adam eplisp")
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--result_file",
default=None,
type=str,
required=False,
help="The result file that the BERT model was trained on.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
# Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument("--do_lower_case",
default=False,
action='store_true',
help="Whether to lower case the input text.")
parser.add_argument(
"--max_seq_length",
default=280,
type=int,
help="maximum total input sequence length after WordPiece tokenization."
)
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--num_labels",
default=1,
type=int,
help="mapping classify nums")
parser.add_argument("--reduce_dim",
default=64,
type=int,
help="from hidden size to this dimensions, reduce dim")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=6.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--weight_decay",
default=0.01,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumualte before")
parser.add_argument(
'--optimize_on_cpu',
default=False,
action='store_true',
help="Whether to perform optimization and averages on CPU")
parser.add_argument(
'--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=128,
help='Loss scale, positive power of 2 can improve fp16 convergence.')
args = parser.parse_args()
data_processor = DataProcessor(args.num_labels)
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info(
"16-bits training currently not supported in distributed training"
)
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
print(f'args.train_batch_size = {args.train_batch_size}')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
# bert_config.reduce_dim = args.reduce_dim
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if args.do_train:
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
def prepare_data(args, task_name='train'):
if task_name == 'train':
file_path = args.train_file
elif task_name == 'eval':
file_path = args.eval_file
elif task_name == 'train_eval':
file_path = args.eval_train_file
elif task_name == 'predict':
file_path = args.predict_file
if os.path.isdir(file_path):
examples = data_processor.read_file_dir(file_path,
top_n=args.top_n)
else:
if task_name == 'predict':
examples, example_map_ids = data_processor.read_predict_examples(
file_path, top_n=args.top_n, task_name=task_name)
else:
examples, example_map_ids = data_processor.read_novel_examples(
file_path, top_n=args.top_n, task_name=task_name)
features = convert_examples_to_features(examples, args.max_seq_length,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features],
dtype=torch.long)
all_example_ids = torch.tensor([f.example_id for f in features],
dtype=torch.long)
if task_name in ['train', 'eval', 'train_eval', 'predict']:
all_label_ids = torch.tensor([f.label_id for f in features],
dtype=torch.long)
datas = TensorDataset(all_example_ids, all_input_ids,
all_input_mask, all_segment_ids,
all_label_ids)
else:
datas = TensorDataset(all_example_ids, all_input_ids,
all_input_mask, all_segment_ids)
if task_name == 'train':
if args.local_rank == -1:
data_sampler = RandomSampler(datas)
else:
data_sampler = DistributedSampler(datas)
dataloader = DataLoader(datas,
sampler=data_sampler,
batch_size=args.train_batch_size,
drop_last=True)
else:
dataloader = DataLoader(datas,
batch_size=args.eval_batch_size,
drop_last=True)
return (dataloader,
example_map_ids) if task_name != 'train' else dataloader
def accuracy(example_ids, logits, probs=None, data_type='eval'):
logits = logits.tolist()
example_ids = example_ids.tolist()
assert len(logits) == len(example_ids)
classify_name = ['no_answer', 'yes_answer']
labels, text_a, novel_names = [], [], []
map_dicts = example_map_ids if data_type == 'eval' else train_example_map_ids
for i in example_ids:
example = map_dicts[i]
labels.append(example.label)
text_a.append("||".join(example.text_a))
novel_names.append(example.name)
write_data = pd.DataFrame({
"text_a": text_a,
"labels": labels,
"logits": logits,
"novel_names": novel_names,
})
write_data['yes_or_no'] = write_data['labels'] == write_data['logits']
if probs is not None:
write_data['logits'] = probs.tolist()
write_data.to_csv(args.result_file, index=False)
assert len(labels) == len(logits)
result = classification_report(labels,
logits,
target_names=classify_name)
return result
def eval_model(model, eval_dataloader, device, data_type='eval'):
model.eval()
eval_loss = 0
all_logits = []
all_example_ids = []
all_probs = []
accuracy_result = None
batch_count = 0
for step, batch in enumerate(tqdm(eval_dataloader, desc="evaluating")):
example_ids, input_ids, input_mask, segment_ids, label_ids = batch
label_ids = label_ids.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
if not args.do_train:
label_ids = None
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids,
segment_ids,
input_mask,
labels=label_ids)
argmax_logits = torch.argmax(logits, dim=1)
first_indices = torch.arange(argmax_logits.size()[0])
logits_probs = logits[first_indices, argmax_logits]
if args.do_train:
eval_loss += tmp_eval_loss.mean().item()
all_logits.append(argmax_logits)
all_example_ids.append(example_ids)
else:
all_logits.append(argmax_logits)
all_example_ids.append(example_ids)
all_probs.append(logits_probs)
batch_count += 1
if all_logits:
all_logits = torch.cat(all_logits, dim=0)
all_example_ids = torch.cat(all_example_ids, dim=0)
all_probs = torch.cat(all_probs, dim=0) if len(all_probs) else None
accuracy_result = accuracy(all_example_ids,
all_logits,
probs=all_probs,
data_type=data_type)
eval_loss /= batch_count
print(f'\n========= {data_type} acc ============')
print(f'{accuracy_result}')
return eval_loss, accuracy_result, all_logits
def predict_model(model, predict_dataloader, device):
model.eval()
all_label_ids = []
all_probs = []
scene_cut_indexs = []
is_cut_probs = []
for step, batch in enumerate(
tqdm(predict_dataloader, desc="predicting")):
example_ids, input_ids, input_mask, segment_ids, label_ids = batch
# print(f'input_ids = {input_ids.shape}')
# print(f'input_mask = {input_mask.shape}')
# print(f'segment_ids = {segment_ids.shape}')
# print(f'label_ids = {label_ids.shape}')
with torch.no_grad():
_, logits = model(input_ids,
segment_ids,
input_mask,
labels=None)
argmax_logits = torch.argmax(logits, dim=1)
first_indices = torch.arange(argmax_logits.size()[0])
logits_probs = logits[first_indices, argmax_logits]
all_label_ids.extend(label_ids.tolist())
all_probs.extend(logits_probs.tolist())
for label, prob, pred_label in zip(label_ids.tolist(),
logits_probs.tolist(),
argmax_logits.tolist()):
if pred_label:
scene_cut_indexs.append(label)
is_cut_probs.append(prob)
# print(f'prob = {prob}')
# print(f'pred label = {pred_label}')
# print(f'sent = {predict_example_map_ids[label]}')
print(f'all_example_ids = {len(all_label_ids)}')
print(f'all_probs = {len(all_probs)}')
assert len(all_label_ids) == len(all_probs)
return scene_cut_indexs, is_cut_probs
if args.do_train:
train_dataloader = prepare_data(args, task_name='train')
num_train_steps = int(
len(train_dataloader) / args.gradient_accumulation_steps *
args.num_train_epochs)
model = BertForSequenceClassification(bert_config,
num_labels=data_processor.num_labels)
new_state_dict = model.state_dict()
init_state_dict = torch.load(
os.path.join(args.bert_model, 'pytorch_model.bin'))
for k, v in init_state_dict.items():
if k in new_state_dict:
print(f'k in = {k} v in shape = {v.shape}')
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
init_model_token_type(model, type_vocab_size=args.top_n)
for k, v in model.state_dict().items():
print(f'k = {k}, v shape {v.shape}')
if args.fp16:
model.half()
if args.do_predict:
model_path = os.path.join(args.output_dir, WEIGHTS_NAME)
new_state_dict = torch.load(model_path)
new_state_dict = dict([(k[7:], v) if k.startswith('module') else (k, v)
for k, v in new_state_dict.items()])
model.load_state_dict(new_state_dict)
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1 and not device == 'cpu':
model = torch.nn.DataParallel(model)
# # model = BalancedDataParallel(1, model, dim=0).to(device)
eval_dataloader, example_map_ids = prepare_data(args, task_name='eval')
train_eval_dataloader, train_example_map_ids = prepare_data(
args, task_name='train_eval')
def get_linear_schedule_with_warmup(optimizer,
num_warmup_steps,
num_training_steps,
last_epoch=-1):
""" Create a schedule with a learning rate that decreases linearly after
linearly increasing during a warmup period.
"""
def lr_lambda(current_step):
if current_step < num_warmup_steps:
return float(current_step) / float(max(1, num_warmup_steps))
return max(
0.0,
float(num_training_steps - current_step) /
float(max(1, num_training_steps - num_warmup_steps)))
return LambdaLR(optimizer, lr_lambda, last_epoch)
def get_optimizers(num_training_steps: int, model):
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
args.weight_decay,
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
warmup_steps = args.warmup_proportion * num_training_steps
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=warmup_steps,
num_training_steps=num_training_steps)
return optimizer, scheduler
if args.do_train:
optimizer, scheduler = get_optimizers(
num_training_steps=num_train_steps, model=model)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
# eval_loss, acc, _ = eval_model(model, eval_dataloader, device)
# logger.info(f'初始开发集loss: {eval_loss}')
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
torch.cuda.empty_cache()
model_save_path = os.path.join(args.output_dir,
f"{WEIGHTS_NAME}.{epoch}")
tr_loss = 0
train_batch_count = 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="training")):
_, input_ids, input_mask, segment_ids, label_ids = batch
label_ids = label_ids.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
loss, _ = model(input_ids,
segment_ids,
input_mask,
labels=label_ids)
if n_gpu > 1:
loss = loss.mean()
if args.fp16 and args.loss_scale != 1.0:
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step()
model.zero_grad()
train_batch_count += 1
tr_loss /= train_batch_count
eval_loss, acc, _ = eval_model(model, eval_dataloader, device)
eval_model(model,
train_eval_dataloader,
device,
data_type='train_eval')
logger.info(
f'训练loss: {tr_loss}, 开发集loss：{eval_loss} 训练轮数：{epoch + 1}/{int(args.num_train_epochs)}'
)
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model.state_dict(), model_save_path)
if epoch == 0:
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
if args.do_predict:
# eval_model(model, train_eval_dataloader, device, data_type='train_eval')
# for root, dir_, files in os.walk(''):
# for file in files:
# arg.predict_file = os.path.join(root, file)
predict_dataloader, predict_example_map_ids = prepare_data(
args, task_name='predict')
scene_cut_indexs, is_cut_probs = predict_model(model,
predict_dataloader,
device)
assert len(scene_cut_indexs) == len(is_cut_probs)
predict_novel_name = os.path.join(
os.path.split(args.predict_file)[0], 'scene_cut_datas',
os.path.split(args.predict_file)[-1])
for i, index in enumerate(scene_cut_indexs):
predict_example_map_ids[
index] = f'######## {round(is_cut_probs[i], 3)} {predict_example_map_ids[index]}'
with open(predict_novel_name, 'w', encoding='utf-8') as f:
for i, line in enumerate(predict_example_map_ids):
f.write(line)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir", default=None, type=str, required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_config_file", default=None, type=str, required=True,
help="The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--vocab_file", default=None, type=str, required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument("--output_dir", default=None, type=str, required=True,
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--init_checkpoint", default=None, type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument("--do_lower_case", default=False, action='store_true',
help="Whether to lower case the input text. True for uncased models, False for cased models.")
parser.add_argument("--max_seq_length", default=128, type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train", default=False, action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval", default=False, action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size", default=32, type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size", default=8, type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate", default=5e-5, type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs", default=3.0, type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion", default=0.1, type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps", default=1000, type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda", default=False, action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank", type=int, default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument("--gpu_ids", type=str, default="0",
help="select one gpu to use")
parser.add_argument('--seed', type=int, default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
help="Number of updates steps to accumualte before performing a backward/update pass.")
parser.add_argument('--optimize_on_cpu', default=False,action='store_true',
help="Whether to perform optimization and keep the optimizer averages on CPU")
parser.add_argument('--fp16', default=False, action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale', type=float, default=128,
help='Loss scaling, positive power of 2 values can improve fp16 convergence.')
################## atec
parser.add_argument('--do_submit', default=False, action='store_true', help="submit to results to atec cloud")
parser.add_argument('--train_devset', default=False, action='store_true', help="")
parser.add_argument("--test_in_file", default=None, type=str, help="")
parser.add_argument("--test_out_file", default=None, type=str, help="")
args = parser.parse_args()
if args.local_rank == -1 and not args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
#device = torch.device("cuda", args.gpu_id)
n_gpu = len(args.gpu_ids.split(','))#torch.cuda.device_count()
elif args.no_cuda:
device = torch.device('cpu')
n_gpu = 0
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info("16-bits training currently not supported in distributed training")
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
#if not args.do_train and not args.do_eval:
# raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format(
args.max_seq_length, bert_config.max_position_embeddings))
#if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir) #, exist_ok=True)
processor = AtecProcessor()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
if args.train_devset:
eval_examples = processor.get_dev_examples(args.data_dir)
train_examples += eval_examples
num_train_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
# Prepare model
logger.info('build model')
model = BertForSequenceClassification(bert_config, len(label_list))
#model = BertSiameseModel(bert_config, len(label_list))
if args.init_checkpoint is not None:
try:
# just model.bert
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
except RuntimeError as e:
# all model
import re
new_state_dict = collections.OrderedDict()
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
for key in state_dict.keys():
new_key = re.sub("module\.", "", key)
new_state_dict[new_key] = state_dict[key]
model.load_state_dict(new_state_dict)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=[int(x) for x in
args.gpu_ids.split(',')])
global_step = 0
tr_loss=None
if args.do_train:
# Prepare optimizer
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
from tqdm import tqdm, trange
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
# train_features = convert_examples_to_siamese_features(
# train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in train_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in train_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in train_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in train_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in train_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in train_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
# train_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
with tqdm(train_dataloader, desc="Iteration") as pbar:
for step, batch in enumerate(pbar):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
# tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch
# loss, logits = model(tokens_a, types_a, mask_a, tokens_b, types_b,
# mask_b, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
preds = th.argmax(logits, dim=1)
acc = accuracy_score(label_ids.detach().cpu().numpy(),
preds.detach().cpu().numpy())
pbar.set_postfix(loss=loss.item(), acc=acc.item())
#nb_tr_examples += input_ids.size(0)
nb_tr_examples += label_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), os.path.join(args.output_dir,
"pytorch_model.bin"))
if args.do_eval:
from tqdm import tqdm, trange
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
# eval_features = convert_examples_to_siamese_features(
# eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
# eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
y_true = []
y_pred = []
for batch in tqdm(eval_dataloader):
# tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch
input_ids, input_mask, segment_ids, label_ids = batch
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
# tmp_eval_loss, logits = model(tokens_a, types_a, mask_a,
# tokens_b, types_b, mask_b, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
preds = np.argmax(logits, axis=1)
y_true.extend(label_ids)
y_pred.extend(preds)
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
#nb_eval_examples += input_ids.size(0)
nb_eval_examples += label_ids.size # np.array
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'precision': precision_score(y_true, y_pred),
'recall': recall_score(y_true, y_pred),
'f1': f1_score(y_true, y_pred)}
if tr_loss is not None:
result['loss'] = tr_loss/nb_tr_steps
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "a") as writer:
logger.info("***** Eval results *****")
writer.write("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_submit:
eval_examples = processor.get_test_examples(args.test_in_file)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
# eval_features = convert_examples_to_siamese_features(
# eval_examples, label_list, args.max_seq_length, tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
#eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
# eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
#eval_sampler = SequentialSampler(eval_data)
#eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
y_pred = []
batch_size = args.eval_batch_size
for i in range(0, len(all_label_ids), batch_size):
input_ids = all_input_ids[i:i+batch_size]
input_mask = all_input_mask[i:i+batch_size]
segment_ids = all_segment_ids[i:i+batch_size]
label_ids = all_label_ids[i:i+batch_size]
# tokens_a = all_tokens_a[i:i+batch_size]
# types_a = all_types_a[i:i+batch_size]
# mask_a = all_mask_a[i:i+batch_size]
# tokens_b = all_tokens_b[i:i+batch_size]
# types_b = all_types_b[i:i+batch_size]
# mask_b = all_mask_b[i:i+batch_size]
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask)
# logits = model(tokens_a, types_a, mask_a,
# tokens_b, types_b, mask_b)
logits = logits.detach().cpu().numpy()
preds = np.argmax(logits, axis=1)
y_pred.extend(preds)
with open(args.test_out_file, "w") as writer:
for i, y in enumerate(y_pred):
writer.write('{}\t{}\n'.format(i+1, y))
def load_model(self, bert_config_file, init_checkpoint, vocab_file,
max_seq_length, task_name, do_lower_case, seed, local_rank,
no_cuda):
logger.info('Calling torch.cuda.current_device [PATCH]')
torch.cuda.current_device()
logger.info(
'[ Local rank: {} ] [ No cuda: {} ] [ Cuda available: {} ]'.format(
local_rank, no_cuda, torch.cuda.is_available()))
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"[ Device: {} ] [ n_gpu: {} ] [ distributed inference: {} ]".
format(device, n_gpu, bool(local_rank != -1)))
logger.info("[ Using seed {} ]".format(seed))
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
logger.info(
"Loading BERT config file from {}".format(bert_config_file))
bert_config = BertConfig.from_json_file(bert_config_file)
if max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(max_seq_length, bert_config.max_position_embeddings))
# prepare dataloaders
processors = {
"sentihood_NLI_B": Sentihood_NLI_B_Processor,
"sentihood_NLI_M": Sentihood_NLI_M_Processor,
}
logger.info("Infering for {} task".format(task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
logger.info(
"Creating tokenizer. [ Vocab file: {} ] [ Lower case: {} ]".format(
vocab_file, do_lower_case))
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
# model and optimizer
model = BertForSequenceClassification(bert_config, len(label_list))
model.load_state_dict(torch.load(init_checkpoint, map_location='cpu'))
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], output_device=local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
return model, tokenizer, processor, device
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
processors = {
"dream": dreamProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
if args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(args.output_dir))
else:
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / n_class / args.train_batch_size /
args.gradient_accumulation_steps * args.num_train_epochs)
model = BertForSequenceClassification(
bert_config, 1 if n_class > 1 else len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in train_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids,
n_class)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), os.path.join(args.output_dir,
"model.pt"))
model.load_state_dict(torch.load(os.path.join(args.output_dir,
"model.pt")))
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if args.do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(args.output_dir,
"eval_results_dev.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(args.output_dir, "logits_dev.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
eval_examples = processor.get_test_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if args.do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(args.output_dir,
"eval_results_test.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(args.output_dir, "logits_test.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
parser.add_argument("--init_checkpoint",
default=None,
type=str,
required=True,
help="The checkpoint file from pretraining")
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--google_pretrained",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--max_steps",
default=-1.0,
type=float,
help="Total number of training steps to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=1,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument("--old",
action='store_true',
help="use old fp16 optimizer")
parser.add_argument(
'--vocab_file',
type=str,
default=None,
required=True,
help="Vocabulary mapping/file BERT was pretrainined on")
parser.add_argument("--config_file",
default=None,
type=str,
required=True,
help="The BERT model config")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
}
num_labels_task = {
"cola": 2,
"mnli": 3,
"mrpc": 2,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
print(
"WARNING: Output directory ({}) already exists and is not empty.".
format(args.output_dir))
if not os.path.exists(args.output_dir) and is_main_process():
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
#tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
tokenizer = BertTokenizer(args.vocab_file,
do_lower_case=args.do_lower_case,
max_len=512) # for bert large
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
config = BertConfig.from_json_file(args.config_file)
# Padding for divisibility by 8
if config.vocab_size % 8 != 0:
config.vocab_size += 8 - (config.vocab_size % 8)
model = BertForSequenceClassification(config, num_labels=num_labels)
print("USING CHECKPOINT from", args.init_checkpoint)
model.load_state_dict(torch.load(args.init_checkpoint,
map_location='cpu')["model"],
strict=False)
print("USED CHECKPOINT from", args.init_checkpoint)
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
print("using fp16")
try:
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False)
if args.loss_scale == 0:
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O2",
keep_batchnorm_fp32=False,
loss_scale="dynamic")
else:
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O2",
keep_batchnorm_fp32=False,
loss_scale=args.loss_scale)
scheduler = LinearWarmUpScheduler(
optimizer,
warmup=args.warmup_proportion,
total_steps=num_train_optimization_steps)
else:
print("using fp32")
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
print("data prep")
cached_train_features_file = args.data_dir + '_{0}_{1}_{2}'.format(
list(filter(None, args.bert_model.split('/'))).pop(),
str(args.max_seq_length), str(args.do_lower_case))
train_features = None
try:
with open(cached_train_features_file, "rb") as reader:
train_features = pickle.load(reader)
except:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
if args.local_rank == -1 or torch.distributed.get_rank() == 0:
logger.info(" Saving train features into cached file %s",
cached_train_features_file)
with open(cached_train_features_file, "wb") as writer:
pickle.dump(train_features, writer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
if args.max_steps > 0 and global_step > args.max_steps:
break
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up for BERT which FusedAdam doesn't do
scheduler.step()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
preds = None
out_label_ids = None
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = label_ids.detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids,
label_ids.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = np.argmax(preds, axis=1)
eval_loss = eval_loss / nb_eval_steps
loss = tr_loss / nb_tr_steps if args.do_train else None
results = {
'eval_loss': eval_loss,
'global_step': global_step,
'loss': loss
}
result = compute_metrics(task_name, preds, out_label_ids)
results.update(result)
print(results)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(results.keys()):
logger.info(" %s = %s", key, str(results[key]))
writer.write("%s = %s\n" % (key, str(results[key])))
def train(args):
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
'sst': SstProcessor,
'polarity': PolarityProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError("Invalid accumulate_gradients parameter: {}, should be >= 1".format(
args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size / args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format(
args.max_seq_length, bert_config.max_position_embeddings))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs)
model = BertForSequenceClassification(bert_config, len(label_list))
if args.do_test:
model.load_state_dict(torch.load(os.path.join(args.data_dir, "model_best.pt")))
model.to(device)
model.eval()
if args.do_eval:
eval_example = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_example, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running eval *****")
logger.info(" Num examples = %d", len(eval_example))
logger.info(" Batch size = %d", args.eval_batch_size)
_ = do_eval(model, device, eval_features, args)
if args.do_predict:
print(os.path.dirname(args.data_dir))
test_example = processor.get_test_examples(os.path.dirname(args.data_dir))
test_features = convert_examples_to_features(
test_example, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running test *****")
logger.info(" Num examples = %d", len(test_example))
logger.info(" Batch size = %d", args.eval_batch_size)
test_logit_list, _ = do_eval(model, device, test_features, args)
np.save(os.path.join(args.data_dir, 'oof_test'), np.asarray(test_logit_list))
return 0
if args.init_checkpoint is not None:
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [
{'params': [p for n, p in model.named_parameters() if n not in no_decay], 'weight_decay_rate': 0.01},
{'params': [p for n, p in model.named_parameters() if n in no_decay], 'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
max_score = 0
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, (input_ids, input_mask, segment_ids, label_ids) in enumerate(
tqdm(train_dataloader, desc="Iteration")):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logit_list = []
labels_eval_list = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
logit_list.extend(logits.tolist())
labels_eval_list.extend(label_ids.tolist())
tmp_eval_accuracy = accuracy(logits, label_ids)
# _ = accuracy2(logits, label_ids)
# _ = accuracy3(logits, label_ids)
# _ = accuracy4(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
# print(epoch)
# f1 = score2(logit_list, labels_eval_list)
eval_loss = eval_loss / nb_eval_steps # len(eval_dataloader)
eval_accuracy = eval_accuracy / nb_eval_examples # len(eval_dataloader)
print("eval_loss", eval_loss)
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps,
'Epoch': epoch,
'Dir': args.data_dir
} # 'loss': loss.item()}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "a") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if eval_accuracy > max_score:
# best_logit_list = logit_list
# best_eval_labels = labels_eval_list
np.save(os.path.join(args.data_dir, 'oof_train'), np.asarray(logit_list))
np.save(os.path.join(args.data_dir, 'oof_train_y'), np.asarray(labels_eval_list))
torch.save(model.state_dict(), os.path.join(args.data_dir, "model_%.4f.pt" % eval_accuracy))
if os.path.exists(os.path.join(args.data_dir, "model_best.pt")):
os.remove(os.path.join(args.data_dir, "model_best.pt"))
torch.save(model.state_dict(), os.path.join(args.data_dir, "model_best.pt"))
max_score = eval_accuracy
def main():
bert_path = bp.bert_path
## parameters
do_lower_case = False
do_train = True
do_eval = True
data_dir = '.'
max_seq_length = 128
train_batch_size = 8
eval_batch_size = 8
learning_rate = 2e-5
num_train_epochs = 8.0
seed = 42
output_dir = 'dream_finetuned'
no_cuda = False
bert_config_file = bert_path + 'bert_config.json'
vocab_file = bert_path + 'vocab.txt'
init_checkpoint = bert_path + 'pytorch_model.bin'
gradient_accumulation_steps = 3
warmup_proportion = 0.1
local_rank = -1
# 用来指定GPU
processors = {
"dream": dreamProcessor,
}
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(local_rank != -1))
if gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(gradient_accumulation_steps))
train_batch_size = int(train_batch_size / gradient_accumulation_steps)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
if not do_train and not do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(bert_config_file)
if max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(output_dir) and os.listdir(output_dir):
if do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(output_dir))
else:
os.makedirs(output_dir, exist_ok=True)
processor = processors['dream']()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
train_examples = None
num_train_steps = None
if do_train:
train_examples = processor.get_train_examples(data_dir)
num_train_steps = int(
len(train_examples) / n_class / train_batch_size /
gradient_accumulation_steps * num_train_epochs)
model = BertForSequenceClassification(
bert_config, 1 if n_class > 1 else len(label_list))
if init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(init_checkpoint, map_location='cpu'))
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], output_device=local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=learning_rate,
warmup=warmup_proportion,
t_total=num_train_steps)
global_step = 0
if do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in train_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=train_batch_size)
model.train()
for _ in trange(int(num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids,
n_class)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if gradient_accumulation_steps > 1:
loss = loss / gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), os.path.join(output_dir, "model.pt"))
model.load_state_dict(torch.load(os.path.join(output_dir, "model.pt")))
if do_eval:
eval_examples = processor.get_dev_examples(data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(output_dir, "eval_results_dev.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(output_dir, "logits_dev.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
eval_examples = processor.get_test_examples(data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(output_dir, "eval_results_test.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(output_dir, "logits_test.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=4,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=666,
help="random seed for initialization")
parser.add_argument('--pooling_ratio',
type=float,
default=0.3,
help='pooling ratio')
parser.add_argument('--pool_dropout_ratio',
type=float,
default=0.5,
help='dropout ratio')
parser.add_argument('--graph_hidden_size',
type=int,
default=300,
help="Hidden_size for graph")
parser.add_argument('--input_dropout',
type=float,
default=0.5,
help='Dropout rate for word representation.')
parser.add_argument('--num_graph_layers',
type=int,
default=2,
help="Number of blocks for graph")
parser.add_argument('--heads',
type=int,
default=3,
help='Num of heads in multi-head attention.')
parser.add_argument('--sublayer_first',
type=int,
default=2,
help='Num of the first sublayers in dcgcn block.')
parser.add_argument('--sublayer_second',
type=int,
default=4,
help='Num of the second sublayers in dcgcn block.')
parser.add_argument('--gcn_dropout',
type=float,
default=0.5,
help='AGGCN layer dropout rate.')
parser.add_argument('--lamada',
type=float,
default=0.000001,
help='Weights for DTW Loss.')
parser.add_argument('--max_offset',
type=int,
default=4,
help='Length of max_offset.')
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=2,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument(
'--optimize_on_cpu',
default=False,
action='store_true',
help=
"Whether to perform optimization and keep the optimizer averages on CPU"
)
parser.add_argument(
'--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=128,
help=
'Loss scaling, positive power of 2 values can improve fp16 convergence.'
)
parser.add_argument("--resume",
default=False,
action='store_true',
help="Whether to resume the training.")
parser.add_argument(
"--f1eval",
default=True,
action='store_true',
help="Whether to use f1 for dev evaluation during training.")
args = parser.parse_args()
processors = {
"bert": bertProcessor,
"bertf1c": bertf1cProcessor,
"berts": bertsProcessor,
"bertsf1c": bertsf1cProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info(
"16-bits training currently not supported in distributed training"
)
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and 'model.pt' in os.listdir(
args.output_dir):
if args.do_train and not args.resume:
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(args.output_dir))
else:
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / n_class / args.train_batch_size /
args.gradient_accumulation_steps * args.num_train_epochs)
model = BertForSequenceClassification(bert_config, 1, args)
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [{
'params': [p for n, p in param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.resume:
model.load_state_dict(
torch.load(os.path.join(args.output_dir, "model.pt")))
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
if args.do_train:
best_metric = 0
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in train_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.float)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids,
1)
if n_gpu > 1:
loss = loss.mean()
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(
param_optimizer,
model.named_parameters(),
test_nan=True)
if is_nan:
logger.info(
"FP16 TRAINING: Nan in gradients, reducing loss scaling"
)
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(
model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, 1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if args.do_train:
result = {
'eval_loss': eval_loss,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss}
if args.f1eval:
eval_f1, eval_T2 = f1_eval(logits_all, eval_features)
result["f1"] = eval_f1
result["T2"] = eval_T2
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
if args.f1eval:
if eval_f1 >= best_metric:
torch.save(model.state_dict(),
os.path.join(args.output_dir, "model_best.pt"))
best_metric = eval_f1
else:
if eval_accuracy >= best_metric:
torch.save(model.state_dict(),
os.path.join(args.output_dir, "model_best.pt"))
best_metric = eval_accuracy
model.load_state_dict(
torch.load(os.path.join(args.output_dir, "model_best.pt")))
torch.save(model.state_dict(), os.path.join(args.output_dir,
"model.pt"))
model.load_state_dict(torch.load(os.path.join(args.output_dir,
"model.pt")))
if args.do_eval:
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, 1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
eval_loss += tmp_eval_loss.mean().item()
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
if args.do_train:
result = {
'eval_loss': eval_loss,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss}
output_eval_file = os.path.join(args.output_dir,
"eval_results_dev.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(args.output_dir, "logits_dev.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
eval_examples = processor.get_test_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, 1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
eval_loss += tmp_eval_loss.mean().item()
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
if args.do_train:
result = {
'eval_loss': eval_loss,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss}
output_eval_file = os.path.join(args.output_dir,
"eval_results_test.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(args.output_dir, "logits_test.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
class LogicLevelModel():
def __init__(self):
self.bert_config = BertConfig.from_json_file(
os.path.join(path, 'uncased_L-12_H-768_A-12/bert_config.json'))
self.max_sequence_length = 128
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
if self.max_sequence_length > self.bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(self.max_sequence_length,
self.bert_config.max_position_embeddings))
self.processor = LogicProcessor()
self.label_list = self.processor.get_labels()
self.tokenizer = tokenization.FullTokenizer(vocab_file=os.path.join(
path, 'uncased_L-12_H-768_A-12/vocab.txt'),
do_lower_case=False)
self.model = BertForSequenceClassification(self.bert_config,
len(self.label_list))
init_checkpoint = os.path.join(path, 'model/logic_model_500.bin')
#Future save model Load code
if init_checkpoint is not None:
self.model.load_state_dict(
torch.load(init_checkpoint, map_location='cpu'))
self.model.to(self.device)
def predict(self, text_a, text_b):
device = self.device
eval_example = self.processor._create_examples(text_a, text_b,
'input example')
eval_feature = convert_examples_to_features(eval_example,
self.label_list,
self.max_sequence_length,
self.tokenizer)
input_ids = torch.tensor([eval_feature.input_ids], dtype=torch.long)
input_mask = torch.tensor([eval_feature.input_mask], dtype=torch.long)
segment_ids = torch.tensor([eval_feature.segment_ids],
dtype=torch.long)
if eval_feature.label_id == None:
label_ids = None
else:
label_ids = torch.tensor(eval_feature.label_ids, dtype=torch.long)
#eval_data = TensorDataset(input_ids, input_mask, segment_ids, label_ids)
#eval_dataloader = DataLoader(eval_data)
self.model.eval()
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
if label_ids == None:
pass
else:
label_ids = label_ids.to(device)
L = []
if label_ids == None:
logits = self.model(input_ids, segment_ids, input_mask, label_ids)
else:
loss, logits = self.model(input_ids, segment_ids, input_mask,
label_ids)
logits = logits.detach().cpu().numpy()
output = np.argmax(logits, axis=1)
output = convert_id_to_label(output, self.label_list)
return output
