def get_predictor_model(cls):
config = BertConfig.from_json_file(config_file)
model = BertForQuestionAnswering(config)
model.load_state_dict(
torch.load(MODEL_PATH, map_location='cpu')["model"])
model.to(device)
cls.model = model
return cls.model
def eval_all():
output_model_file = "../model_dir/best_model"
output_config_file = "../model_dir/bert_config.json"
config = BertConfig(output_config_file)
model = BertForQuestionAnswering(config)
model.load_state_dict(
torch.load(output_model_file)) #, map_location='cpu'))
evaluate(model.cpu(), result_file="../metric/predicts.json")
def initialize_model(args):
''' return model, ready to trace '''
config = BertConfig.from_json_file(args.config_file)
if config.vocab_size % 8 != 0:
config.vocab_size += 8 - (config.vocab_size % 8)
model = BertForQuestionAnswering(config)
model.enable_apex(False)
state_dict = torch.load(args.checkpoint, map_location='cpu')["model"]
model.load_state_dict(state_dict)
if args.fp16:
model.half()
return model
def eval_all():
# output_model_file = "../../output/best_model"
output_model_file = MODEL_PATH
output_config_file = os.path.join('../model_dir/', args.config_name)
config = BertConfig(output_config_file)
model = BertForQuestionAnswering(config)
if not args.no_pai:
try:
model.load_state_dict(torch.load(output_model_file))#, map_location='cpu'))
except:
model = nn.DataParallel(model)
model.load_state_dict(torch.load(output_model_file))#, map_location='cpu'))
else:
try:
model.load_state_dict(torch.load(output_model_file, map_location='cpu'))
except:
model = nn.DataParallel(model)
model.load_state_dict(torch.load(output_model_file, map_location='cpu'))
result_file_path = os.path.join('../metric', args.result_file_name)
evaluate(model, result_file=result_file_path)
if not args.no_pai:
print(os.getcwd())
pai_file_output = "/Container/thsi_yicui/dureader-bert/Dureader/output"
client.upload(pai_file_output, result_file_path, overwrite=True)
def eval_all():
output_model_file = "../model_dir/best_model"
output_config_file = "../model_dir/bert_configbase.json"
config = BertConfig(output_config_file)
model = BertForQuestionAnswering(config)
# 针对多卡训练加载模型的方法：
state_dict = torch.load(output_model_file, map_location='cuda:0')
# 初始化一个空 dict
new_state_dict = OrderedDict()
# 修改 key，没有module字段则需要不上，如果有，则需要修改为 module.features
for k, v in state_dict.items():
if 'module' not in k:
k = k
else:
k = k.replace('module.', '')
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
# model.load_state_dict(torch.load(output_model_file)) #, map_location='cpu'))
evaluate(model.cpu(), result_file="../metric/predicts_dev.json")
def main():
parser = argparse.ArgumentParser()
BERT_DIR = "./model/uncased_L-12_H-768_A-12/"
## Required parameters
parser.add_argument("--bert_config_file", default=BERT_DIR+"bert_config.json", \
type=str, help="The config json file corresponding to the pre-trained BERT model. "
"This specifies the model architecture.")
parser.add_argument("--vocab_file", default=BERT_DIR+"vocab.txt", type=str, \
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument("--output_dir", default="out", type=str, \
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--train_file", type=str, \
help="SQuAD json for training. E.g., train-v1.1.json", \
default="")
parser.add_argument("--predict_file", type=str,
help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json", \
default="")
parser.add_argument("--init_checkpoint", type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).", \
default=BERT_DIR+"pytorch_model.bin")
parser.add_argument(
"--do_lower_case",
default=True,
action='store_true',
help="Whether to lower case the input text. Should be True for uncased "
"models and False for cased models.")
parser.add_argument(
"--max_seq_length",
default=300,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
parser.add_argument(
"--max_query_length",
default=64,
type=int,
help=
"The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--predict_batch_size",
default=128,
type=int,
help="Total batch size for predictions.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=10.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% "
"of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--iterations_per_loop",
default=1000,
type=int,
help="How many steps to make in each estimator call.")
parser.add_argument(
"--n_best_size",
default=3,
type=int,
help=
"The total number of n-best predictions to generate in the nbest_predictions.json "
"output file.")
parser.add_argument(
"--max_answer_length",
default=30,
type=int,
help=
"The maximum length of an answer that can be generated. This is needed because the start "
"and end predictions are not conditioned on one another.")
parser.add_argument(
"--verbose_logging",
default=False,
action='store_true',
help=
"If true, all of the warnings related to data processing will be printed. "
"A number of warnings are expected for a normal SQuAD evaluation.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument('--eval_period', type=int, default=2000)
parser.add_argument('--max_n_answers', type=int, default=5)
parser.add_argument('--merge_query', type=int, default=-1)
parser.add_argument('--reduce_layers', type=int, default=-1)
parser.add_argument('--reduce_layers_to_tune', type=int, default=-1)
parser.add_argument('--only_comp', action="store_true", default=False)
parser.add_argument('--train_subqueries_file', type=str, default="") #500
parser.add_argument('--predict_subqueries_file', type=str,
default="") #500
parser.add_argument('--prefix', type=str, default="") #500
parser.add_argument('--model', type=str, default="qa") #500
parser.add_argument('--pooling', type=str, default="max")
parser.add_argument('--debug', action="store_true", default=False)
parser.add_argument('--output_dropout_prob', type=float, default=0)
parser.add_argument('--wait_step', type=int, default=30)
parser.add_argument('--with_key', action="store_true", default=False)
parser.add_argument('--add_noise', action="store_true", default=False)
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_file:
raise ValueError(
"If `do_train` is True, then `train_file` must be specified.")
if not args.predict_file:
raise ValueError(
"If `do_train` is True, then `predict_file` must be specified."
)
if args.do_predict:
if not args.predict_file:
raise ValueError(
"If `do_predict` is True, then `predict_file` must be specified."
)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.do_train and args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
logger.info("Output directory () already exists and is not empty.")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
eval_dataloader, eval_examples, eval_features, _ = get_dataloader(
logger=logger,
args=args,
input_file=args.predict_file,
subqueries_file=args.predict_subqueries_file,
is_training=False,
batch_size=args.predict_batch_size,
num_epochs=1,
tokenizer=tokenizer)
if args.do_train:
train_dataloader, train_examples, _, num_train_steps = get_dataloader(
logger=logger, args=args, \
input_file=args.train_file, \
subqueries_file=args.train_subqueries_file, \
is_training=True,
batch_size=args.train_batch_size,
num_epochs=args.num_train_epochs,
tokenizer=tokenizer)
#a = input()
if args.model == 'qa':
model = BertForQuestionAnswering(bert_config, 4)
metric_name = "F1"
elif args.model == 'classifier':
if args.reduce_layers != -1:
bert_config.num_hidden_layers = args.reduce_layers
model = BertClassifier(bert_config, 2, args.pooling)
metric_name = "F1"
elif args.model == "span-predictor":
if args.reduce_layers != -1:
bert_config.num_hidden_layers = args.reduce_layers
if args.with_key:
Model = BertForQuestionAnsweringWithKeyword
else:
Model = BertForQuestionAnswering
model = Model(bert_config, 2)
metric_name = "Accuracy"
else:
raise NotImplementedError()
if args.init_checkpoint is not None and args.do_predict and \
len(args.init_checkpoint.split(','))>1:
assert args.model == "qa"
model = [model]
for i, checkpoint in enumerate(args.init_checkpoint.split(',')):
if i > 0:
model.append(BertForQuestionAnswering(bert_config, 4))
print("Loading from", checkpoint)
state_dict = torch.load(checkpoint, map_location='cpu')
filter = lambda x: x[7:] if x.startswith('module.') else x
state_dict = {filter(k): v for (k, v) in state_dict.items()}
model[-1].load_state_dict(state_dict)
model[-1].to(device)
else:
if args.init_checkpoint is not None:
print("Loading from", args.init_checkpoint)
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
if args.reduce_layers != -1:
state_dict = {k:v for k, v in state_dict.items() \
if not '.'.join(k.split('.')[:3]) in \
['encoder.layer.{}'.format(i) for i in range(args.reduce_layers, 12)]}
if args.do_predict:
filter = lambda x: x[7:] if x.startswith('module.') else x
state_dict = {filter(k): v for (k, v) in state_dict.items()}
model.load_state_dict(state_dict)
else:
model.bert.load_state_dict(state_dict)
if args.reduce_layers_to_tune != -1:
model.bert.embeddings.required_grad = False
n_layers = 12 if args.reduce_layers == -1 else args.reduce_layers
for i in range(n_layers - args.reduce_layers_to_tune):
model.bert.encoder.layer[i].require_grad = False
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.do_train:
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params':
[p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
best_f1 = 0
wait_step = 0
model.train()
global_step = 0
stop_training = False
for epoch in range(int(args.num_train_epochs)):
for step, batch in tqdm(enumerate(train_dataloader)):
global_step += 1
batch = [t.to(device) for t in batch]
loss = model(batch, global_step)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
if global_step % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
if global_step % args.eval_period == 0:
model.eval()
f1 = predict(args, model, eval_dataloader, eval_examples, eval_features, \
device, write_prediction=False)
logger.info("%s: %.3f on epoch=%d" %
(metric_name, f1 * 100.0, epoch))
if best_f1 < f1:
logger.info("Saving model with best %s: %.3f -> %.3f on epoch=%d" % \
(metric_name, best_f1*100.0, f1*100.0, epoch))
model_state_dict = {
k: v.cpu()
for (k, v) in model.state_dict().items()
}
torch.save(
model_state_dict,
os.path.join(args.output_dir, "best-model.pt"))
model = model.cuda()
best_f1 = f1
wait_step = 0
stop_training = False
else:
wait_step += 1
if best_f1 > 0.1 and wait_step == args.wait_step:
stop_training = True
model.train()
if stop_training:
break
elif args.do_predict:
if type(model) == list:
model = [m.eval() for m in model]
else:
model.eval()
f1 = predict(args, model, eval_dataloader, eval_examples,
eval_features, device)
logger.info("Final %s score: %.3f%%" % (metric_name, f1 * 100.0))
def function_main_bert(q_id, context_ans, question):
bert_config_file = 'bert_config.json'
vocab_file = 'vocab.txt'
# output_dir='output'
# processed_data = 'processed'
#predict_file='ASQdev.json'
finetuned_checkpoint = 'ft_model_bert.bin'
max_seq_length = 500
do_lower_case = True
local_rank = -1
seed = 42
n_best_size = 20
predict_batch_size = 8
max_answer_length = 500
max_query_length = 64
doc_stride = 128
max_seq_length = 500
final_answer = ""
probs = 0.0
no_cuda = False
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
bert_config = BertConfig.from_json_file(bert_config_file)
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
eval_examples = read_squad_examples(id=q_id,
paragraph=context_ans,
question=question,
tokenizer=tokenizer)
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=max_seq_length,
doc_stride=doc_stride,
max_query_length=max_query_length)
model = BertForQuestionAnswering(bert_config)
state_dict = torch.load(finetuned_checkpoint, map_location='cpu')
new_state_dict = collections.OrderedDict()
for key, value in state_dict.items():
new_state_dict[key[:]] = value
model.load_state_dict(new_state_dict)
del state_dict
del new_state_dict
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], output_device=local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
final_answer, probs, start_logits, end_logits, eval_feature = run_evaluate(
local_rank, predict_batch_size, n_best_size, max_answer_length,
do_lower_case, model, eval_features, device, eval_examples, tokenizer)
return final_answer, probs, start_logits, end_logits, eval_feature
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--init_checkpoint",
default=None,
type=str,
required=True,
help="The checkpoint file from pretraining")
## Other parameters
parser.add_argument(
"--verbose_logging",
action='store_true',
help=
"If true, all of the warnings related to data processing will be printed. "
)
parser.add_argument("--seed", default=1, type=int)
parser.add_argument(
"--question",
default=
"Most antibiotics target bacteria and don't affect what class of organisms? ",
type=str,
help="question")
parser.add_argument(
"--context",
default=
"Within the genitourinary and gastrointestinal tracts, commensal flora serve as biological barriers by competing with pathogenic bacteria for food and space and, in some cases, by changing the conditions in their environment, such as pH or available iron. This reduces the probability that pathogens will reach sufficient numbers to cause illness. However, since most antibiotics non-specifically target bacteria and do not affect fungi, oral antibiotics can lead to an overgrowth of fungi and cause conditions such as a vaginal candidiasis (a yeast infection). There is good evidence that re-introduction of probiotic flora, such as pure cultures of the lactobacilli normally found in unpasteurized yogurt, helps restore a healthy balance of microbial populations in intestinal infections in children and encouraging preliminary data in studies on bacterial gastroenteritis, inflammatory bowel diseases, urinary tract infection and post-surgical infections. ",
type=str,
help="context")
parser.add_argument(
"--max_seq_length",
default=384,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument(
"--max_query_length",
default=64,
type=int,
help=
"The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument(
"--n_best_size",
default=1,
type=int,
help="The total number of n-best predictions to generate. ")
parser.add_argument(
"--max_answer_length",
default=30,
type=int,
help=
"The maximum length of an answer that can be generated. This is needed because the start "
"and end predictions are not conditioned on one another.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument(
"--do_lower_case",
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
'--version_2_with_negative',
action='store_true',
help='If true, then the model can reply with "unknown". ')
parser.add_argument(
'--null_score_diff_threshold',
type=float,
default=-11.0,
help=
"If null_score - best_non_null is greater than the threshold predict 'unknown'. "
)
parser.add_argument(
'--vocab_file',
type=str,
default=None,
required=True,
help="Vocabulary mapping/file BERT was pretrainined on")
parser.add_argument("--config_file",
default=None,
type=str,
required=True,
help="The BERT model config")
parser.add_argument('--fp16',
action='store_true',
help="use mixed-precision")
parser.add_argument("--local_rank",
default=-1,
help="ordinal of the GPU to use")
args = parser.parse_args()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
tokenizer = BertTokenizer(args.vocab_file,
do_lower_case=args.do_lower_case,
max_len=512) # for bert large
# Prepare model
config = BertConfig.from_json_file(args.config_file)
# Padding for divisibility by 8
if config.vocab_size % 8 != 0:
config.vocab_size += 8 - (config.vocab_size % 8)
# initialize model
model = BertForQuestionAnswering(config)
model.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu')["model"])
model.to(device)
if args.fp16:
model.half()
model.eval()
print("question: ", args.question)
print("context: ", args.context)
print()
# preprocessing
doc_tokens = args.context.split()
query_tokens = tokenizer.tokenize(args.question)
feature = preprocess_tokenized_text(doc_tokens,
query_tokens,
tokenizer,
max_seq_length=args.max_seq_length,
max_query_length=args.max_query_length)
tensors_for_inference, tokens_for_postprocessing = feature
input_ids = torch.tensor(tensors_for_inference.input_ids,
dtype=torch.long).unsqueeze(0)
segment_ids = torch.tensor(tensors_for_inference.segment_ids,
dtype=torch.long).unsqueeze(0)
input_mask = torch.tensor(tensors_for_inference.input_mask,
dtype=torch.long).unsqueeze(0)
# load tensors to device
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
# run prediction
with torch.no_grad():
start_logits, end_logits = model(input_ids, segment_ids, input_mask)
# post-processing
start_logits = start_logits[0].detach().cpu().tolist()
end_logits = end_logits[0].detach().cpu().tolist()
answer, answers = get_answer(doc_tokens, tokens_for_postprocessing,
start_logits, end_logits, args)
# print result
print()
print(answer)
print()
print(json.dumps(answers, indent=4))
def main():
parser = argparse.ArgumentParser()
BERT_DIR = "uncased_L-12_H-768_A-12/"
## Required parameters
parser.add_argument("--bert_config_file", default=BERT_DIR+"bert_config.json", \
type=str, help="The config json file corresponding to the pre-trained BERT model. "
"This specifies the model architecture.")
parser.add_argument("--vocab_file", default=BERT_DIR+"vocab.txt", type=str, \
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument("--output_dir", default="out", type=str, \
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--load", default=False, action='store_true')
parser.add_argument("--train_file", type=str, \
help="SQuAD json for training. E.g., train-v1.1.json", \
default="/home/sewon/data/squad/train-v1.1.json")
parser.add_argument("--predict_file", type=str,
help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json", \
default="/home/sewon/data/squad/dev-v1.1.json")
parser.add_argument("--init_checkpoint", type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).", \
default=BERT_DIR+"pytorch_model.bin")
parser.add_argument(
"--do_lower_case",
default=True,
action='store_true',
help="Whether to lower case the input text. Should be True for uncased "
"models and False for cased models.")
parser.add_argument(
"--max_seq_length",
default=300,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
parser.add_argument(
"--max_query_length",
default=64,
type=int,
help=
"The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=39,
type=int,
help="Total batch size for training.")
parser.add_argument("--predict_batch_size",
default=300,
type=int,
help="Total batch size for predictions.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=1000.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% "
"of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--iterations_per_loop",
default=1000,
type=int,
help="How many steps to make in each estimator call.")
parser.add_argument(
"--n_best_size",
default=3,
type=int,
help=
"The total number of n-best predictions to generate in the nbest_predictions.json "
"output file.")
parser.add_argument(
"--verbose_logging",
default=False,
action='store_true',
help=
"If true, all of the warnings related to data processing will be printed. "
"A number of warnings are expected for a normal SQuAD evaluation.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument('--eval_period', type=int, default=500)
parser.add_argument('--max_n_answers', type=int, default=20)
parser.add_argument('--n_paragraphs', type=str, default='40')
parser.add_argument('--verbose', action="store_true", default=False)
parser.add_argument('--wait_step', type=int, default=12)
# Learning method variation
parser.add_argument('--loss_type', type=str, default="mml")
parser.add_argument('--tau', type=float, default=12000.0)
# For evaluation
parser.add_argument('--prefix', type=str, default="") #500
parser.add_argument('--debug', action="store_true", default=False)
args = parser.parse_args()
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
print("Output directory () already exists and is not empty.")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO,
handlers=[
logging.FileHandler(os.path.join(args.output_dir, "log.txt")),
logging.StreamHandler()
])
logger = logging.getLogger(__name__)
logger.info(args)
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_file:
raise ValueError(
"If `do_train` is True, then `train_file` must be specified.")
if not args.predict_file:
raise ValueError(
"If `do_train` is True, then `predict_file` must be specified."
)
if args.do_predict:
if not args.predict_file:
raise ValueError(
"If `do_predict` is True, then `predict_file` must be specified."
)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.do_train and args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(args.max_seq_length, bert_config.max_position_embeddings))
model = BertForQuestionAnswering(bert_config,
device,
4,
loss_type=args.loss_type,
tau=args.tau)
metric_name = "EM"
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
train_split = ',' in args.train_file
if train_split:
n_train_files = len(args.train_file.split(','))
eval_dataloader, eval_examples, eval_features, _ = get_dataloader(
logger=logger,
args=args,
input_file=args.predict_file,
is_training=False,
batch_size=args.predict_batch_size,
num_epochs=1,
tokenizer=tokenizer)
if args.do_train:
train_file = args.train_file
if train_split:
train_file = args.train_file.split(',')[0]
train_dataloader, _, _, num_train_steps = get_dataloader(
logger=logger, args=args, \
input_file=train_file, \
is_training=True,
batch_size=args.train_batch_size,
num_epochs=args.num_train_epochs,
tokenizer=tokenizer)
if args.init_checkpoint is not None:
logger.info("Loading from {}".format(args.init_checkpoint))
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
if args.do_train and args.init_checkpoint.endswith(
'pytorch_model.bin'):
model.bert.load_state_dict(state_dict)
else:
filter = lambda x: x[7:] if x.startswith('module.') else x
state_dict = {filter(k): v for (k, v) in state_dict.items()}
model.load_state_dict(state_dict)
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.do_train:
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params':
[p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
best_f1 = (-1, -1)
wait_step = 0
model.train()
global_step = 0
stop_training = False
train_losses = []
for epoch in range(int(args.num_train_epochs)):
if epoch > 0 and train_split:
train_file = args.train_file.split(',')[epoch % n_train_files]
train_dataloader = get_dataloader(
logger=logger, args=args, \
input_file=train_file, \
is_training=True,
batch_size=args.train_batch_size,
num_epochs=args.num_train_epochs,
tokenizer=tokenizer)[0]
for step, batch in enumerate(train_dataloader):
global_step += 1
batch = [t.to(device) for t in batch]
loss = model(batch, global_step)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
train_losses.append(loss.detach().cpu())
loss.backward()
if global_step % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
if global_step % args.eval_period == 0:
model.eval()
f1 = predict(logger, args, model, eval_dataloader, eval_examples, eval_features, \
device, write_prediction=False)
logger.info(
"Step %d Train loss %.2f EM %.2f F1 %.2f on epoch=%d" %
(global_step, np.mean(train_losses), f1[0] * 100,
f1[1] * 100, epoch))
train_losses = []
if best_f1 < f1:
logger.info("Saving model with best %s: %.2f (F1 %.2f) -> %.2f (F1 %.2f) on epoch=%d" % \
(metric_name, best_f1[0]*100, best_f1[1]*100, f1[0]*100, f1[1]*100, epoch))
model_state_dict = {
k: v.cpu()
for (k, v) in model.state_dict().items()
}
torch.save(
model_state_dict,
os.path.join(args.output_dir, "best-model.pt"))
model = model.to(device)
best_f1 = f1
wait_step = 0
stop_training = False
else:
wait_step += 1
if wait_step == args.wait_step:
stop_training = True
model.train()
if stop_training:
break
logger.info("Training finished!")
elif args.do_predict:
if type(model) == list:
model = [m.eval() for m in model]
else:
model.eval()
f1 = predict(logger,
args,
model,
eval_dataloader,
eval_examples,
eval_features,
device,
varying_n_paragraphs=len(args.n_paragraphs) > 1)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--bert_config_file",
default='data/bert_config.json',
type=str,
help="The config json file corresponding to the pre-trained BERT model. "
"This specifies the model architecture.")
parser.add_argument(
"--vocab_file",
default='data/vocab.txt',
type=str,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default='output',
type=str,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument(
"--processed_data",
default='processed',
type=str,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
## Other parameters
parser.add_argument("--train_file",
default='data/squad_train.json',
type=str,
help="SQuAD json for training. E.g., train-v1.1.json")
parser.add_argument(
"--predict_file",
default='data/squad_dev.json',
type=str,
help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json"
)
parser.add_argument(
"--init_checkpoint",
default='data/pytorch_model.bin',
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--finetuned_checkpoint",
default='ft_dir/ft_model.bin',
type=str,
help="finetuned checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=True,
action='store_true',
help="Whether to lower case the input text. Should be True for uncased "
"models and False for cased models.")
parser.add_argument(
"--max_seq_length",
default=500,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
parser.add_argument(
"--max_query_length",
default=64,
type=int,
help=
"The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument(
"--max_answer_length",
default=500,
type=int,
help=
"The maximum length of an answer that can be generated. This is needed because the start "
"and end predictions are not conditioned on one another.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--predict_batch_size",
default=8,
type=int,
help="Total batch size for predictions.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=5.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% "
"of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--iterations_per_loop",
default=1000,
type=int,
help="How many steps to make in each estimator call.")
parser.add_argument(
"--n_best_size",
default=20,
type=int,
help=
"The total number of n-best predictions to generate in the nbest_predictions.json "
"output file.")
parser.add_argument(
"--verbose_logging",
default=False,
action='store_true',
help=
"If true, all of the warnings related to data processing will be printed. "
"A number of warnings are expected for a normal SQuAD evaluation.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_file:
raise ValueError(
"If `do_train` is True, then `train_file` must be specified.")
if args.do_predict:
if not args.predict_file:
raise ValueError(
"If `do_predict` is True, then `predict_file` must be specified."
)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory () already exists and is not empty.")
os.makedirs(args.output_dir, exist_ok=True)
if not os.path.exists(args.ft_dir):
os.makedirs(args.ft_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
num_train_steps = None
if args.do_train:
logger.info('Load and process train examples')
if os.path.exists(
os.path.join(args.processed_data, 'processed_train.pkl')):
with open(os.path.join(args.processed_data, 'processed_train.pkl'),
'rb') as f:
train_features, train_examples, num_train_steps = pickle.load(
f)
else:
train_examples = read_squad_examples(input_file=args.train_file,
tokenizer=tokenizer,
is_training=True)
num_train_steps = int(
len(train_examples) / args.train_batch_size *
args.num_train_epochs)
train_features = convert_examples_to_features(
examples=train_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=True)
train_path = os.path.join(args.processed_data,
'processed_train.pkl')
with open(train_path, 'wb') as f:
pickle.dump([train_features, train_examples, num_train_steps],
f)
if args.do_predict:
logger.info('Load and process dev examples')
if os.path.exists(
os.path.join(args.processed_data, 'processed_dev.pkl')):
with open(os.path.join(args.processed_data, 'processed_dev.pkl'),
'rb') as f:
eval_features, eval_examples = pickle.load(f)
else:
eval_examples = read_squad_examples(input_file=args.predict_file,
tokenizer=tokenizer,
is_training=False)
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=False)
eval_path = os.path.join(args.processed_data, 'processed_dev.pkl')
with open(eval_path, 'wb') as f:
pickle.dump([eval_features, eval_examples], f)
model = BertForQuestionAnswering(bert_config)
if args.do_train and args.init_checkpoint is not None:
logger.info('Loading init checkpoint')
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
logger.info('Loaded init checkpoint')
elif args.do_predict:
logger.info('Loading fine-tuned checkpoint')
state_dict = torch.load(args.finetuned_checkpoint, map_location='cpu')
new_state_dict = collections.OrderedDict()
for key, value in state_dict.items():
new_state_dict[key[7:]] = value
model.load_state_dict(new_state_dict)
del state_dict
del new_state_dict
logger.info('Loaded fine-tuned checkpoint')
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num split examples = %d", len(train_features))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_start_positions = torch.tensor(
[f.start_position for f in train_features], dtype=torch.long)
all_end_positions = torch.tensor(
[f.end_position for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_start_positions,
all_end_positions)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
best_dev_score = 0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, start_positions, end_positions = batch
loss = model(input_ids, segment_ids, input_mask,
start_positions, end_positions)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if (step + 1) % args.save_checkpoints_steps == 0:
best_dev_score = run_evaluate(args, model, eval_features,
device, eval_examples,
tokenizer, best_dev_score)
logger.info('Best dev score {} in steps {}:'.format(
best_dev_score, step))
if args.do_predict:
run_evaluate(args, model, eval_features, device, eval_examples,
tokenizer)