np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
tokenizer = BertTokenizer.from_pretrained(bert_model,
do_lower_case=do_lower_case)
num_train_optimization_steps = int(
len(train_InputExamples) / batch_size /
gradient_accumulation_steps) * num_epochs
model_qa = BertQA.from_pretrained(
bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
if args.local_rank == 0:
torch.distributed.barrier()
model_qa.to(device)
train_features = bert_utils.convert_examples_to_features(
train_InputExamples, MAX_SEQ_LENGTH, tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
print()
batch_size = batch_size // gradient_accumulation_steps
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
tokenizer = BertTokenizer.from_pretrained(args.output_dir,
do_lower_case=do_lower_case)
model_qa = BertQA.from_pretrained(args.output_dir)
model_qa.to(device)
dev_features = bert_utils.convert_examples_to_features(dev_InputExamples,
MAX_SEQ_LENGTH,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in dev_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in dev_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in dev_features],
dtype=torch.long)
all_start_positions = torch.tensor([f.start_label_ids for f in dev_features],
dtype=torch.long)
all_end_positions = torch.tensor([f.end_label_ids for f in dev_features],
dtype=torch.long)