def get_bert_embedding(self,
textlist,
tags,
file_name,
model_dir='out/',
model_config="model_config.json"):
model_config = os.path.join(model_dir, model_config)
model_config = json.load(open(model_config))
output_config_file = os.path.join(model_dir, CONFIG_NAME)
output_model_file = os.path.join(model_dir, WEIGHTS_NAME)
config = BertConfig(output_config_file)
model = BertNer(config, num_labels=model_config["num_labels"])
dict_embeddings = {}
#token2tag={}
n_txt = 0
for text in textlist:
#print(n_txt)
_, logits = self.predict(text)
# print(log#its.size())
if n_txt % 1000 == 0:
print('{}/{}'.format(n_txt, len(textlist)))
input_ids, input_mask, segment_ids, valid_ids, tokens = self.preprocess(
text)
input_ids = torch.tensor([input_ids], dtype=torch.long)
input_mask = torch.tensor([input_mask], dtype=torch.long)
segment_ids = torch.tensor([segment_ids], dtype=torch.long)
valid_ids = torch.tensor([valid_ids], dtype=torch.long)
bert_embed, _ = model.bert(input_ids,
token_type_ids=None,
attention_mask=None,
output_all_encoded_layers=False)
tensor_idx = 0
#print(valid_ids)
#print(tokens)
#print(tags[n_txt])
#print('boucle 2')
#print(valid_ids)
for i in range(len(tokens)):
#print(tensor_idx-1)
if (valid_ids[0][i] != 0) and (tokens[i] != '[CLS]') and (
tokens[i] != '[SEP]'):
tensor_idx += 1
#print(tensor_idx-1)
#print(tokens[i])
l_tag = tags[n_txt][tensor_idx - 1].split("/")
if len(l_tag) > 2:
tag = l_tag[1] + '/' + l_tag[2]
if (tag in dict_embeddings):
#token2embedding[tokens[i]]=average2list(bert_embed[0,tensor_idx,:].tolist(),token2embedding[tokens[i]])
#print('if\n'+tags[n_txt][tensor_idx-1])
#if tags[n_txt][tensor_idx-1] not in token2tag[tokens[i]]:
# token2tag[tokens[i]].append(tags[n_txt][tensor_idx-1])
dict_embeddings[tag].append(
bert_embed[0, tensor_idx, :].tolist() +
logits[0, tensor_idx, 1:9].tolist())
#dict_embeddings[tags[n_txt][tensor_idx-1]].append(bert_embed[0,tensor_idx,:].tolist())
elif (tag not in dict_embeddings):
#token2embedding[tokens[i]]=bert_embed[0,tensor_idx,:].tolist()
#print('else\n'+tags[n_txt][tensor_idx-1])
#token2tag[tokens[i]]=[tags[n_txt][tensor_idx-1]]
dict_embeddings[tag] = [
bert_embed[0, tensor_idx, :].tolist() +
logits[0, tensor_idx, 1:9].tolist()
]
n_txt += 1
with open(file_name, 'w') as f:
json.dump(dict_embeddings, f)
return dict_embeddings
def main():
args = parse_args()
# specifies the path where the biobert or clinical bert model is saved
if args.bert_model == 'biobert' or args.bert_model == 'clinical_bert':
args.bert_model = args.model_loc
print(f"Using bert model: {args.bert_model}")
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
logger.info(f"device: {device} n_gpu: {n_gpu}")
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
processor = N2c2ClsProcessor(args.fold_id)
num_labels = 13
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
print('TRAIN')
train = processor.get_train_examples(args.data_dir)
print([(train[i].text_a, train[i].text_b, train[i].label)
for i in range(3)])
print('DEV')
dev = processor.get_dev_examples(args.data_dir)
print([(dev[i].text_a, dev[i].text_b, dev[i].label) for i in range(3)])
print('TEST')
test = processor.get_test_examples(args.data_dir)
print([(test[i].text_a, test[i].text_b, test[i].label) for i in range(3)])
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) /
args.train_batch_size) * args.num_train_epochs
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else PYTORCH_PRETRAINED_BERT_CACHE
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
pred = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = torch.softmax(logits, 1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
pred += logits.tolist()
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
pred = {f.guid: p for f, p in zip(eval_features, pred)}
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_pred_file = os.path.join(args.output_dir, "pred_results.txt")
with open(output_pred_file, 'w') as writer:
logger.info("***** Writing Eval predictions *****")
for id, p in pred.items():
writer.write(f"{id}:{p}\n")
if args.do_test and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running testing *****")
logger.info(" Num examples = %d", len(test_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=args.eval_batch_size)
model.eval()
test_loss, test_accuracy = 0, 0
nb_test_steps, nb_test_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
test_dataloader, desc="Testing"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_test_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_test_accuracy = accuracy(logits, label_ids)
test_loss += tmp_test_loss.mean().item()
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'test_loss': test_loss,
'test_accuracy': test_accuracy,
'global_step': global_step,
'loss': loss
}
output_test_file = os.path.join(args.output_dir, "test_results.txt")
with open(output_test_file, "w") as writer:
logger.info("***** Test results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .csv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
#if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
#raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = read_swag_examples(os.path.join(
args.data_dir, 'train.csv'),
is_training=True)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = os.path.join(PYTORCH_PRETRAINED_BERT_CACHE,
'distributed_{}'.format(args.local_rank))
predictor = BertForMultipleChoice.from_pretrained(args.bert_model,
cache_dir=cache_dir,
num_choices=4)
if args.fp16:
predictor.half()
predictor.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
predictor = DDP(predictor)
elif n_gpu > 1:
predictor = torch.nn.DataParallel(predictor)
# Prepare optimizer
param_optimizer_pred = list(predictor.named_parameters())
# hack to remove pooler, which is not used
# thus it produce None grad that break apex
param_optimizer_pred = [
n for n in param_optimizer_pred if 'pooler' not in n[0]
]
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters_pred = [{
'params': [
p for n, p in param_optimizer_pred
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params': [
p for n, p in param_optimizer_pred
if any(nd in n for nd in no_decay)
],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer_pred = FusedAdam(optimizer_grouped_parameters_pred,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer_pred = FP16_Optimizer(optimizer_pred,
dynamic_loss_scale=True)
else:
optimizer_pred = FP16_Optimizer(optimizer_pred,
static_loss_scale=args.loss_scale)
else:
optimizer_pred = BertAdam(optimizer_grouped_parameters_pred,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
tokenizer,
args.max_seq_length,
True)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor(select_field(train_features, 'input_ids'),
dtype=torch.long)
all_input_mask = torch.tensor(select_field(train_features,
'input_mask'),
dtype=torch.long)
all_segment_ids = torch.tensor(select_field(train_features,
'segment_ids'),
dtype=torch.long)
all_label = torch.tensor([f.label for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
training_history = []
predictor.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss_pred = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss_pred, _ = predictor(input_ids, segment_ids, input_mask,
label_ids)
if n_gpu > 1:
loss_pred = loss_pred.mean(
) # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss_pred = loss_pred * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss_pred = loss_pred / args.gradient_accumulation_steps
tr_loss_pred += loss_pred.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
training_history.append([loss_pred.item()])
if args.fp16:
optimizer_pred.backward(loss_pred)
else:
loss_pred.backward()
# if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer_pred.param_groups:
param_group['lr'] = lr_this_step
optimizer_pred.step()
optimizer_pred.zero_grad()
global_step += 1
history_file = open(os.path.join(args.output_dir, "train_results.csv"),
"w")
writer = csv.writer(history_file, delimiter=",")
writer.writerow(["pred_loss"])
for row in training_history:
writer.writerow(row)
if args.do_train:
# Save a trained model and the associated configuration
model_to_save = predictor.module if hasattr(
predictor, 'module') else predictor # Only save the model it-self
WEIGHTS_NAME = 'weights.pt'
CONFIG_NAME = 'config.json'
output_model_file = os.path.join(args.output_dir,
'predictor_' + WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir,
'predictor_' + CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
predictor = BertForMultipleChoice(config, num_choices=4)
predictor.load_state_dict(torch.load(output_model_file))
else:
WEIGHTS_NAME = 'weights.pt'
CONFIG_NAME = 'config.json'
output_model_file = os.path.join(args.output_dir,
'predictor_' + WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir,
'predictor_' + CONFIG_NAME)
config = BertConfig(output_config_file)
predictor = BertForMultipleChoice(config, num_choices=4)
predictor.load_state_dict(torch.load(output_model_file))
predictor.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = read_swag_examples(os.path.join(
args.data_dir, 'val.csv'),
is_training=True)
eval_features = convert_examples_to_features(eval_examples, tokenizer,
args.max_seq_length, True)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'),
dtype=torch.long)
all_input_mask = torch.tensor(select_field(eval_features,
'input_mask'),
dtype=torch.long)
all_segment_ids = torch.tensor(select_field(eval_features,
'segment_ids'),
dtype=torch.long)
all_label = torch.tensor([f.label for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
predictor.eval()
eval_loss_pred, eval_accuracy_pred = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss_pred, logits_pred = predictor(
input_ids, segment_ids, input_mask, label_ids)
# print("logits_adv", logits_adv)
tmp_eval_accuracy_pred = accuracy(logits_pred, label_ids)
eval_loss_pred += tmp_eval_loss_pred.mean().item()
eval_accuracy_pred += tmp_eval_accuracy_pred.item()
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss_pred /= nb_eval_steps
eval_accuracy_pred /= nb_eval_examples
if args.do_train:
result = {
'eval_loss_pred': eval_loss_pred,
'eval_accuracy_pred': eval_accuracy_pred,
'global_step': global_step,
'loss_pred': tr_loss_pred / nb_tr_steps
}
else:
result = {
'eval_loss_pred': eval_loss_pred,
'eval_accuracy_pred': eval_accuracy_pred
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
global tokenizer, test_collater, model
logger.info('Launching the MT-DNN training')
opt = vars(args)
# update data dir
opt['data_dir'] = data_dir
batch_size = args.batch_size
tasks = {}
tasks_class = {}
nclass_list = []
decoder_opts = []
task_types = []
dropout_list = []
loss_types = []
kd_loss_types = []
#train_datasets = []
for dataset in args.train_datasets:
prefix = dataset.split('_')[0]
if prefix in tasks: continue
assert prefix in task_defs.n_class_map
assert prefix in task_defs.data_type_map
data_type = task_defs.data_type_map[prefix]
nclass = task_defs.n_class_map[prefix]
task_id = len(tasks)
if args.mtl_opt > 0:
task_id = tasks_class[nclass] if nclass in tasks_class else len(tasks_class)
task_type = task_defs.task_type_map[prefix]
dopt = generate_decoder_opt(task_defs.enable_san_map[prefix], opt['answer_opt'])
if task_id < len(decoder_opts):
decoder_opts[task_id] = min(decoder_opts[task_id], dopt)
else:
decoder_opts.append(dopt)
task_types.append(task_type)
loss_types.append(task_defs.loss_map[prefix])
kd_loss_types.append(task_defs.kd_loss_map[prefix])
if prefix not in tasks:
tasks[prefix] = len(tasks)
if args.mtl_opt < 1: nclass_list.append(nclass)
if (nclass not in tasks_class):
tasks_class[nclass] = len(tasks_class)
if args.mtl_opt > 0: nclass_list.append(nclass)
dropout_p = task_defs.dropout_p_map.get(prefix, args.dropout_p)
dropout_list.append(dropout_p)
train_path = os.path.join(data_dir, '{}_train.json'.format(dataset))
logger.info('Loading {} as task {}'.format(train_path, task_id))
# train_data_set = SingleTaskDataset(train_path, True, maxlen=args.max_seq_len, task_id=task_id,
# task_type=task_type, data_type=data_type)
# train_datasets.append(train_data_set)
#train_collater = Collater(dropout_w=args.dropout_w, encoder_type=encoder_type)
# multi_task_train_dataset = MultiTaskDataset(train_datasets)
# multi_task_batch_sampler = MultiTaskBatchSampler(train_datasets, args.batch_size, args.mix_opt, args.ratio)
# multi_task_train_data = DataLoader(multi_task_train_dataset, batch_sampler=multi_task_batch_sampler,
# collate_fn=train_collater.collate_fn, pin_memory=args.cuda)
opt['answer_opt'] = decoder_opts
opt['task_types'] = task_types
opt['tasks_dropout_p'] = dropout_list
opt['loss_types'] = loss_types
opt['kd_loss_types'] = kd_loss_types
args.label_size = ','.join([str(l) for l in nclass_list])
logger.info(args.label_size)
dev_data_list = []
test_data_list = []
test_collater = Collater(is_train=False, encoder_type=encoder_type)
logger.info('#' * 20)
logger.info(opt)
logger.info('#' * 20)
bert_model_path = 'checkpoints/my_mnli/model_0.pt'
state_dict = None
if encoder_type == EncoderModelType.BERT:
if os.path.exists(bert_model_path):
state_dict = torch.load(bert_model_path, map_location=torch.device('cpu'))
config = state_dict['config']
config['attention_probs_dropout_prob'] = args.bert_dropout_p
config['hidden_dropout_prob'] = args.bert_dropout_p
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
else:
logger.error('#' * 20)
logger.error('Could not find the init model!\n The parameters will be initialized randomly!')
logger.error('#' * 20)
config = BertConfig(vocab_size_or_config_json_file=30522).to_dict()
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
elif encoder_type == EncoderModelType.ROBERTA:
bert_model_path = '{}/model.pt'.format(bert_model_path)
if os.path.exists(bert_model_path):
new_state_dict = {}
state_dict = torch.load(bert_model_path)
for key, val in state_dict['model'].items():
if key.startswith('decoder.sentence_encoder'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
elif key.startswith('classification_heads'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
state_dict = {'state': new_state_dict}
model = MTDNNModel(opt, state_dict=state_dict)
if args.resume and args.model_ckpt:
logger.info('loading model from {}'.format(args.model_ckpt))
model.load(args.model_ckpt)
#### model meta str
headline = '############# Model Arch of MT-DNN #############'
### print network
logger.info('\n{}\n{}\n'.format(headline, model.network))
# dump config
config_file = os.path.join(output_dir, 'config.json')
with open(config_file, 'w', encoding='utf-8') as writer:
writer.write('{}\n'.format(json.dumps(opt)))
writer.write('\n{}\n{}\n'.format(headline, model.network))
logger.info("Total number of params: {}".format(model.total_param))
# # tensorboard
# if args.tensorboard:
# args.tensorboard_logdir = os.path.join(args.output_dir, args.tensorboard_logdir)
# tensorboard = SummaryWriter(log_dir=args.tensorboard_logdir)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--bert_model_or_config_file",
default=None,
type=str,
required=True,
help=
"Directory containing pre-trained BERT model or path of configuration file (if no pre-training)."
)
parser.add_argument("--train_file",
default=None,
type=str,
required=True,
help="The input train corpus.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--learning_rate",
default=3e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument(
"--on_memory",
action='store_true',
help="Whether to load train samples into memory or use disk")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
"--num_gpus",
type=int,
default=-1,
help="Num GPUs to use for training (0 for none, -1 for all available)")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
# Check whether bert_model_or_config_file is a file or directory
if os.path.isdir(args.bert_model_or_config_file):
pretrained = True
targets = [WEIGHTS_NAME, CONFIG_NAME, "tokenizer.pkl"]
for t in targets:
path = os.path.join(args.bert_model_or_config_file, t)
if not os.path.exists(path):
msg = "File '{}' not found".format(path)
raise ValueError(msg)
fp = os.path.join(args.bert_model_or_config_file, CONFIG_NAME)
config = BertConfig(fp)
else:
pretrained = False
config = BertConfig(args.bert_model_or_config_file)
# What GPUs do we use?
if args.num_gpus == -1:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
device_ids = None
else:
device = torch.device("cuda" if torch.cuda.is_available()
and args.num_gpus > 0 else "cpu")
n_gpu = args.num_gpus
if n_gpu > 1:
device_ids = list(range(n_gpu))
if args.local_rank != -1:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
# Check some other args
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
if not args.do_train:
raise ValueError(
"Training is currently the only implemented execution option. Please set `do_train`."
)
# Seed RNGs
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
# Prepare output directory
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# Make tokenizer
if pretrained:
fp = os.path.join(args.bert_model_or_config_file, "tokenizer.pkl")
with open(fp, "rb") as f:
tokenizer = pickle.load(f)
else:
training_data = [
line.strip() for line in open(args.train_file).readlines()
]
tokenizer = CuneiformCharTokenizer(training_data=training_data)
tokenizer.trim_vocab(config.min_freq)
# Adapt vocab size in config
config.vocab_size = len(tokenizer.vocab)
print("Size of vocab: {}".format(len(tokenizer.vocab)))
# Get training data
num_train_optimization_steps = None
if args.do_train:
print("Loading Train Dataset", args.train_file)
train_dataset = BERTDataset(args.train_file,
tokenizer,
seq_len=args.max_seq_length,
corpus_lines=None,
on_memory=args.on_memory)
num_train_optimization_steps = int(
len(train_dataset) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
if pretrained:
model = BertForMaskedLM.from_pretrained(args.bert_model_or_config_file)
else:
model = BertForMaskedLM(config)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=device_ids)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
# Prepare training log
output_log_file = os.path.join(args.output_dir, "training_log.txt")
with open(output_log_file, "w") as f:
f.write("Steps\tTrainLoss\n")
# Start training
global_step = 0
total_tr_steps = 0
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
if args.local_rank == -1:
train_sampler = RandomSampler(train_dataset)
else:
#TODO: check if this works with current data generator from disk that relies on next(file)
# (it doesn't return item back by index)
train_sampler = DistributedSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, lm_label_ids = batch
loss = model(input_ids, segment_ids, input_mask, lm_label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
avg_loss = tr_loss / nb_tr_examples
# Update training log
total_tr_steps += nb_tr_steps
log_data = [str(total_tr_steps), "{:.5f}".format(avg_loss)]
with open(output_log_file, "a") as f:
f.write("\t".join(log_data) + "\n")
# Save model
logger.info("** ** * Saving model ** ** * ")
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
fn = os.path.join(args.output_dir, "tokenizer.pkl")
with open(fn, "wb") as f:
pickle.dump(tokenizer, f)
def main():
logger.info('Launching the MT-DNN training')
opt = vars(args)
# update data dir
opt['data_dir'] = data_dir
batch_size = args.batch_size
train_data_list = []
tasks = {}
tasks_class = {}
nclass_list = []
decoder_opts = []
dropout_list = []
for dataset in args.train_datasets:
prefix = dataset.split('_')[0]
if prefix in tasks: continue
assert prefix in DATA_META
assert prefix in DATA_TYPE
data_type = DATA_TYPE[prefix]
nclass = DATA_META[prefix]
task_id = len(tasks)
if args.mtl_opt > 0:
task_id = tasks_class[nclass] if nclass in tasks_class else len(
tasks_class)
task_type = TASK_TYPE[prefix]
pw_task = False
if prefix in opt['pw_tasks']:
pw_task = True
dopt = generate_decoder_opt(prefix, opt['answer_opt'])
if task_id < len(decoder_opts):
decoder_opts[task_id] = min(decoder_opts[task_id], dopt)
else:
decoder_opts.append(dopt)
if prefix not in tasks:
tasks[prefix] = len(tasks)
if args.mtl_opt < 1: nclass_list.append(nclass)
if (nclass not in tasks_class):
tasks_class[nclass] = len(tasks_class)
if args.mtl_opt > 0: nclass_list.append(nclass)
dropout_p = args.dropout_p
if tasks_config and prefix in tasks_config:
dropout_p = tasks_config[prefix]
dropout_list.append(dropout_p)
train_path = os.path.join(data_dir, '{}_train.json'.format(dataset))
logger.info('Loading {} as task {}'.format(train_path, task_id))
train_data = BatchGen(BatchGen.load(train_path,
True,
pairwise=pw_task,
maxlen=args.max_seq_len),
batch_size=batch_size,
dropout_w=args.dropout_w,
gpu=args.cuda,
task_id=task_id,
maxlen=args.max_seq_len,
pairwise=pw_task,
data_type=data_type,
task_type=task_type)
train_data_list.append(train_data)
opt['answer_opt'] = decoder_opts
opt['tasks_dropout_p'] = dropout_list
args.label_size = ','.join([str(l) for l in nclass_list])
logger.info(args.label_size)
dev_data_list = []
test_data_list = []
for dataset in args.test_datasets:
prefix = dataset.split('_')[0]
task_id = tasks_class[
DATA_META[prefix]] if args.mtl_opt > 0 else tasks[prefix]
task_type = TASK_TYPE[prefix]
pw_task = False
if prefix in opt['pw_tasks']:
pw_task = True
assert prefix in DATA_TYPE
data_type = DATA_TYPE[prefix]
dev_path = os.path.join(data_dir, '{}_dev.json'.format(dataset))
dev_data = None
if os.path.exists(dev_path):
dev_data = BatchGen(BatchGen.load(dev_path,
False,
pairwise=pw_task,
maxlen=args.max_seq_len),
batch_size=args.batch_size_eval,
gpu=args.cuda,
is_train=False,
task_id=task_id,
maxlen=args.max_seq_len,
pairwise=pw_task,
data_type=data_type,
task_type=task_type)
dev_data_list.append(dev_data)
test_path = os.path.join(data_dir, '{}_test.json'.format(dataset))
test_data = None
if os.path.exists(test_path):
test_data = BatchGen(BatchGen.load(test_path,
False,
pairwise=pw_task,
maxlen=args.max_seq_len),
batch_size=args.batch_size_eval,
gpu=args.cuda,
is_train=False,
task_id=task_id,
maxlen=args.max_seq_len,
pairwise=pw_task,
data_type=data_type,
task_type=task_type)
test_data_list.append(test_data)
logger.info('#' * 20)
logger.info(opt)
logger.info('#' * 20)
all_iters = [iter(item) for item in train_data_list]
all_lens = [len(bg) for bg in train_data_list]
num_all_batches = args.epochs * sum(all_lens)
if len(train_data_list) > 1 and args.ratio > 0:
num_all_batches = int(args.epochs * (len(train_data_list[0]) *
(1 + args.ratio)))
model_path = args.init_checkpoint
state_dict = None
if os.path.exists(model_path):
state_dict = torch.load(model_path)
config = BertConfig.from_json_file(
"/Users/sai/Downloads/pubmed_pmc_470k/bert_config.json")
# config = state_dict['config']
config = config.to_dict()
config['attention_probs_dropout_prob'] = args.bert_dropout_p
config['hidden_dropout_prob'] = args.bert_dropout_p
opt.update(config)
else:
logger.error('#' * 20)
logger.error(
'Could not find the init model!\n The parameters will be initialized randomly!'
)
logger.error('#' * 20)
config = BertConfig(vocab_size_or_config_json_file=30522).to_dict()
opt.update(config)
model = MTDNNModel(opt,
state_dict=state_dict,
num_train_step=num_all_batches)
####model meta str
headline = '############# Model Arch of MT-DNN #############'
###print network
logger.info('\n{}\n{}\n'.format(headline, model.network))
import pdb
pdb.set_trace()
# dump config
config_file = os.path.join(output_dir, 'config.json')
with open(config_file, 'w', encoding='utf-8') as writer:
writer.write('{}\n'.format(json.dumps(opt)))
writer.write('\n{}\n{}\n'.format(headline, model.network))
logger.info("Total number of params: {}".format(model.total_param))
if args.freeze_layers > 0:
model.network.freeze_layers(args.freeze_layers)
if args.cuda:
model.cuda()
for epoch in range(0, args.epochs):
logger.warning('At epoch {}'.format(epoch))
for train_data in train_data_list:
train_data.reset()
start = datetime.now()
all_indices = []
if len(train_data_list) > 1 and args.ratio > 0:
main_indices = [0] * len(train_data_list[0])
extra_indices = []
for i in range(1, len(train_data_list)):
extra_indices += [i] * len(train_data_list[i])
random_picks = int(
min(len(train_data_list[0]) * args.ratio, len(extra_indices)))
extra_indices = np.random.choice(extra_indices,
random_picks,
replace=False)
if args.mix_opt > 0:
extra_indices = extra_indices.tolist()
random.shuffle(extra_indices)
all_indices = extra_indices + main_indices
else:
all_indices = main_indices + extra_indices.tolist()
else:
for i in range(1, len(train_data_list)):
all_indices += [i] * len(train_data_list[i])
if args.mix_opt > 0:
random.shuffle(all_indices)
all_indices += [0] * len(train_data_list[0])
if args.mix_opt < 1:
random.shuffle(all_indices)
for i in range(len(all_indices)):
task_id = all_indices[i]
batch_meta, batch_data = next(all_iters[task_id])
model.update(batch_meta, batch_data)
if (model.updates
) % args.log_per_updates == 0 or model.updates == 1:
logger.info(
'Task [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]'
.format(
task_id, model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) *
(len(all_indices) - i - 1)).split('.')[0]))
for idx, dataset in enumerate(args.test_datasets):
prefix = dataset.split('_')[0]
label_dict = GLOBAL_MAP.get(prefix, None)
dev_data = dev_data_list[idx]
if dev_data is not None:
dev_metrics, dev_predictions, scores, golds, dev_ids = eval_model(
model, dev_data, dataset=prefix, use_cuda=args.cuda)
for key, val in dev_metrics.items():
logger.warning(
"Task {0} -- epoch {1} -- Dev {2}: {3:.3f}".format(
dataset, epoch, key, val))
score_file = os.path.join(
output_dir, '{}_dev_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': dev_metrics,
'predictions': dev_predictions,
'uids': dev_ids,
'scores': scores
}
dump(score_file, results)
official_score_file = os.path.join(
output_dir, '{}_dev_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
# test eval
test_data = test_data_list[idx]
if test_data is not None:
test_metrics, test_predictions, scores, golds, test_ids = eval_model(
model,
test_data,
dataset=prefix,
use_cuda=args.cuda,
with_label=False)
score_file = os.path.join(
output_dir,
'{}_test_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': test_metrics,
'predictions': test_predictions,
'uids': test_ids,
'scores': scores
}
dump(score_file, results)
official_score_file = os.path.join(
output_dir, '{}_test_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
logger.info('[new test scores saved.]')
model_file = os.path.join(output_dir, 'model_{}.pt'.format(epoch))
model.save(model_file)
from data_load import HParams
from ner_model import Net
from pytorch_pretrained_bert.modeling import BertConfig
from pytorch_pretrained_bert import BertModel
import parameters
import numpy as np
import torch
config = BertConfig(vocab_size_or_config_json_file=parameters.BERT_CONFIG_FILE)
def build_model(config, state_dict, hp):
model = Net(config, vocab_len=len(hp.VOCAB), bert_state_dict=None)
_ = model.load_state_dict(torch.load(state_dict, map_location='cpu'))
_ = model.to('cpu') # inference
return model
i2b2_model = build_model(config, parameters.I2b2_WEIGHTS, HParams('i2b2'))
# Process Query
def process_query(query, hp, model):
s = query
split_s = ["[CLS]"] + s.split() + ["[SEP]"]
x = [] # list of ids
is_heads = [] # list. 1: the token is the first piece of a word
for w in split_s:
tokens = hp.tokenizer.tokenize(w) if w not in ("[CLS]",
"[SEP]") else [w]
def main(config, save_name, label_list, logger, load_model_path=False):
# if not os.path.exists(config.cache_dir + save_name):
# os.makedirs(config.cache_dir + save_name)
# Bert 模型输出文件
output_model_file = os.path.join(config.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(config.output_dir, CONFIG_NAME)
# 设备准备
gpu_ids = [int(device_id) for device_id in config.gpu_ids.split()]
device, n_gpu = get_device(gpu_ids[0])
if n_gpu > 1:
n_gpu = len(gpu_ids)
config.train_batch_size = config.train_batch_size // config.gradient_accumulation_steps
# 设定随机种子
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(config.seed)
# 数据准备
tokenizer = BertTokenizer.from_pretrained(
config.bert_vocab_file, do_lower_case=config.do_lower_case) # 分词器选择
num_labels = len(label_list)
# Train and dev
if config.do_train:
train_dataloader, train_examples_len = load_data(
config.data_dir, tokenizer, config.max_seq_length,
config.train_batch_size, "train", label_list)
dev_dataloader, _ = load_data(config.data_dir, tokenizer,
config.max_seq_length,
config.dev_batch_size, "dev", label_list)
num_train_optimization_steps = int(
train_examples_len / config.train_batch_size /
config.gradient_accumulation_steps) * config.num_train_epochs
# 模型准备
if not load_model_path:
logger.info("create model#########")
if config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [
int(val) for val in config.filter_sizes.split()
]
model = BertCNN.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
model.to(device)
else:
logger.info("loading model##############")
bert_config = BertConfig(output_config_file)
if config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [
int(val) for val in config.filter_sizes.split()
]
model = BertCNN(bert_config,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
model.load_state_dict(
torch.load(output_model_file,
map_location=torch.device('cpu')))
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=gpu_ids)
""" 优化器准备 """
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=config.warmup_proportion,
t_total=num_train_optimization_steps)
""" 损失函数准备 """
criterion = nn.NLLLoss()
criterion = criterion.to(device)
train(config.num_train_epochs, n_gpu, model, train_dataloader,
dev_dataloader, optimizer, criterion,
config.gradient_accumulation_steps, device, label_list,
output_model_file, output_config_file, config.log_dir,
config.print_step, config.early_stop, logger)
""" Test """
# test 数据
test_dataloader, _ = load_data(config.data_dir, tokenizer,
config.max_seq_length,
config.test_batch_size, "test", label_list)
# 加载模型
bert_config = BertConfig(output_config_file)
if config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNN(bert_config,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
model.load_state_dict(
torch.load(output_model_file, map_location=torch.device("cpu")))
model.to(device)
# 损失函数准备
criterion = nn.NLLLoss()
criterion = criterion.to(device)
# test the model
test_loss, test_acc, test_report, test_auc, all_idx, all_labels, all_preds = evaluate_save(
model, test_dataloader, criterion, device, label_list)
logger.info("-------------- Test -------------")
logger.info(
f'\t Loss: {test_loss: .3f} | Acc: {test_acc*100: .3f} % | AUC:{test_auc}'
)
for label in label_list:
logger.info('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
print_list = ['macro avg', 'weighted avg']
for label in print_list:
logger.info('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
calcu_metric(all_labels, all_preds)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_model", default=None, type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.")
## Other parameters
parser.add_argument("--cache_dir",
default="",
type=str,
help="Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument("--max_seq_length",
default=128,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion",
default=0.1,
type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument('--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale',
type=float, default=0,
help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mnli-mm": MnliMismatchedProcessor,
"mrpc": MrpcProcessor,
"sst-2": Sst2Processor,
"sts-b": StsbProcessor,
"qqp": QqpProcessor,
"qnli": QnliProcessor,
"rte": RteProcessor,
"wnli": WnliProcessor,
"adlhw2": MyTaskProcessor
}
output_modes = {
"cola": "classification",
"mnli": "classification",
"mrpc": "classification",
"sst-2": "classification",
"sts-b": "regression",
"qqp": "classification",
"qnli": "classification",
"rte": "classification",
"wnli": "classification",
"adlhw2": "classification"
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size()
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank))
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
print("output_model_file: ", output_model_file)
print("output_config_file: ", output_config_file)
print("Load the config!!!")
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
#model = BertForSequenceClassification.from_pretrained(args.bert_model,
# cache_dir=cache_dir,
# num_labels=num_labels)
#print("model_1: ", model)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer, output_mode)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
# define a new function to compute loss values for both output_modes
logits = model(input_ids, segment_ids, input_mask, labels=None)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(global_step/num_train_optimization_steps, args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
#output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
#output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
#print("output_model_file: ", output_model_file)
#print("output_config_file: ", output_config_file)
#print("Load the config!!!")
print("model", model)
#config = BertConfig(output_config_file)
#model = BertForSequenceClassification(config, num_labels=num_labels)
#model.load_state_dict(torch.load(output_model_file))
#model= torch.load("/home/tzutengweng/ADLHW/A2/code_new/bert_output_1/pytorch_model.bin")
#print("model", model)
#input()
model.to(device)
#print("model: ", model)
if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
#print("eval_examples: ", eval_examples) #a list of <__main__.InputExample object at 0x7f61b67bef28>
#input()
Ids = [e.guid for e in eval_examples]
#print("Ids: ", Ids) #a list of ids
#input()
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
#print("eval_features: ", eval_features) #a list of <__main__.InputFeatures object at 0x7f86f15cb400>
#input()
logger.info("***** Running testing *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
#print("input_ids: ", input_ids)
#print("input_mask: ", input_mask)
#print("segment_ids: ", segment_ids)
print("labels_ids: ", label_ids) #labels_ids: tensor([1, 1, 1, 1, 1, 1, 1, 1], device='cuda:0'
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
print("logits: ", logits.shape)
print("logits: ", logits)
input()
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
#print("logits: ", logits.detach().cpu().numpy())
#input()
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
#print("preds: ", preds)
#print("preds[0]: ", preds[0])
#input()
#numpy.append: Append values to the end of an array.
#https://docs.scipy.org/doc/numpy/reference/generated/numpy.append.html
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
#print("preds: ", preds.shape) #preds: (2210, 5)
#input()
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
#print("preds: ", preds) #[2 4 3 ... 4 1 0] returning the index, predicted label will be index+1
#print("preds: ", preds.shape) #preds: (2210,)
elif output_mode == "regression":
preds = np.squeeze(preds)
result = compute_metrics(task_name, preds, all_label_ids.numpy())
loss = tr_loss/nb_tr_steps if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir, "test.csv")
output_test_file_1 = os.path.join(args.output_dir, "test_results.txt")
# with open(output_test_file_1, "w") as writer:
# logger.info("***** Storing test results *****")
# for key in sorted(result.keys()):
# logger.info(" %s = %s", key, str(result[key]))
# writer.write("%s = %s\n" % (key, str(result[key])))
with open(output_eval_file, "w") as f:
logger.info("***** Test results *****")
writer = csv.DictWriter(f, fieldnames=['Id', 'label'])
writer.writeheader()
writer.writerows(
[{'Id': Id, 'label': p + 1} for Id, p in zip(Ids, preds)])
# hack for MNLI-MM
if task_name == "mnli":
task_name = "mnli-mm"
processor = processors[task_name]()
if os.path.exists(args.output_dir + '-MM') and os.listdir(args.output_dir + '-MM') and args.do_train:
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir + '-MM'):
os.makedirs(args.output_dir + '-MM')
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
preds = np.argmax(preds, axis=1)
result = compute_metrics(task_name, preds, all_label_ids.numpy())
loss = tr_loss/nb_tr_steps if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir + '-MM', "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--input_file", default=None, type=str, required=True)
parser.add_argument("--output_file", default=None, type=str, required=True)
parser.add_argument("--bert_model", default=None, type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese.")
## Other parameters
parser.add_argument("--pretrained_squad_model", default=None, type=str)
parser.add_argument("--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.")
parser.add_argument("--layers", default="-1,-2,-3,-4", type=str)
parser.add_argument("--max_seq_length", default=128, type=int,
help="The maximum total input sequence length after WordPiece tokenization. Sequences longer "
"than this will be truncated, and sequences shorter than this will be padded.")
parser.add_argument("--batch_size", default=32, type=int, help="Batch size for predictions.")
parser.add_argument("--local_rank",
type=int,
default=-1,
help = "local_rank for distributed training on gpus")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device: {} n_gpu: {} distributed training: {}".format(device, n_gpu, bool(args.local_rank != -1)))
layer_indexes = [int(x) for x in args.layers.split(",")]
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
examples = read_examples(args.input_file)
features = convert_examples_to_features(
examples=examples, seq_length=args.max_seq_length, tokenizer=tokenizer)
unique_id_to_feature = {}
for feature in features:
unique_id_to_feature[feature.unique_id] = feature
if args.pretrained_squad_model:
input_config_file = os.path.join(args.pretrained_squad_model, CONFIG_NAME)
input_model_file = os.path.join(args.pretrained_squad_model, WEIGHTS_NAME)
config = BertConfig(input_config_file)
qa_model = BertForQuestionAnswering(config)
qa_model.load_state_dict(torch.load(input_model_file, map_location=device))
model = qa_model.bert # The model we will use for extracting
else:
model = BertModel.from_pretrained(args.bert_model)
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_example_index)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.batch_size)
model.eval()
with open(args.output_file, "w", encoding='utf-8') as writer:
for input_ids, input_mask, example_indices in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
all_encoder_layers, _ = model(input_ids, token_type_ids=None, attention_mask=input_mask)
all_encoder_layers = all_encoder_layers
for b, example_index in enumerate(example_indices):
feature = features[example_index.item()]
unique_id = int(feature.unique_id)
# feature = unique_id_to_feature[unique_id]
output_json = collections.OrderedDict()
output_json["linex_index"] = unique_id
all_out_features = []
for (i, token) in enumerate(feature.tokens):
all_layers = []
for (j, layer_index) in enumerate(layer_indexes):
layer_output = all_encoder_layers[int(layer_index)].detach().cpu().numpy()
layer_output = layer_output[b]
layers = collections.OrderedDict()
layers["index"] = layer_index
layers["values"] = [
round(x.item(), 6) for x in layer_output[i]
]
all_layers.append(layers)
out_features = collections.OrderedDict()
out_features["token"] = token
out_features["layers"] = all_layers
all_out_features.append(out_features)
output_json["features"] = all_out_features
writer.write(json.dumps(output_json) + "\n")
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model checkpoints and predictions will be written."
)
## Other parameters
parser.add_argument("--train_file", default=None, type=str)
parser.add_argument("--predict_file", default=None, type=str)
parser.add_argument(
"--max_seq_length",
default=384,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--predict_batch_size",
default=8,
type=int,
help="Total batch size for predictions.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% "
"of training.")
parser.add_argument(
"--verbose_logging",
action='store_true',
help=
"If true, all of the warnings related to data processing will be printed."
)
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
"--do_lower_case",
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument(
'--evaluate_each_epoch',
action='store_true',
help="Whether to run the evaluation script after every training epoch")
parser.add_argument(
'--early_stopping',
action='store_true',
help=
"Whether to stop finetuning of F1 score on validation set does not improve"
)
parser.add_argument('--train_languages',
nargs='+',
help='<Required> Finetuning languages',
required=False)
parser.add_argument('--predict_languages',
nargs='+',
help='Validation/prediction languages',
required=False)
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_file:
raise ValueError(
"If `do_train` is True, then `train_file` must be specified.")
if args.do_predict:
if not args.predict_file:
raise ValueError(
"If `do_predict` is True, then `predict_file` must be specified."
)
if os.path.exists(args.output_dir) and len(os.listdir(
args.output_dir)) > 1 and args.do_train:
raise ValueError(
"Output directory () already exists and is not empty.")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
tensorboard_writer = SummaryWriter(os.path.join(args.output_dir, "runs"))
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = read_ner_examples(input_file=args.train_file,
is_training=True,
languages=args.train_languages)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
model = AdversarialBertForNER.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)),
num_labels=len(train_examples[0].label_vocab) if train_examples else 1,
num_languages=len(args.train_languages or args.predict_languages))
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
# hack to remove pooler, which is not used
# thus it produce None grad that break apex
param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
if args.do_predict and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = read_ner_examples(input_file=args.predict_file,
is_training=False,
languages=args.predict_languages)
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
is_training=False,
languages=args.predict_languages)
input_filename = os.path.basename(args.predict_file).replace(
".lang", "." + "_".join(args.predict_languages))
output_filepath = os.path.join(args.output_dir,
input_filename + ".predictions.txt")
def evaluate_model(model):
logger.info("***** Running predictions *****")
logger.info(" Num orig examples = %d", len(eval_examples))
logger.info(" Num split examples = %d", len(eval_features))
logger.info(" Batch size = %d", args.predict_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor(
[f.input_mask for f in eval_features], dtype=torch.long)
all_loss_mask = torch.tensor([f.loss_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor(
[f.segment_ids for f in eval_features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0),
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_loss_mask, all_segment_ids,
all_example_index)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.predict_batch_size)
model.eval()
all_results = []
logger.info("Start evaluating")
for input_ids, input_mask, loss_mask, segment_ids, example_indices in tqdm(
eval_dataloader, desc="Evaluating"):
if len(all_results) % 1000 == 0:
logger.info("Processing example: %d" % (len(all_results)))
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
loss_mask = loss_mask.to(device)
segment_ids = segment_ids.to(device)
with torch.no_grad():
batch_logits = model(input_ids, segment_ids, input_mask,
loss_mask)
for i, example_index in enumerate(example_indices):
logits = batch_logits[i].detach().cpu()
eval_feature = eval_features[example_index.item()]
unique_id = int(eval_feature.unique_id)
all_results.append(
RawResult(unique_id=unique_id, logits=logits))
return write_predictions(eval_examples, eval_features, all_results,
output_filepath, args.verbose_logging)
else:
def evaluate_model(model):
pass
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
global_step = 0
if args.do_train:
cached_train_features_file = args.train_file + '_{0}_{1}'.format(
list(filter(None, args.bert_model.split('/'))).pop(),
str(args.max_seq_length))
train_features = None
try:
with open(cached_train_features_file, "rb") as reader:
train_features = pickle.load(reader)
except:
train_features = convert_examples_to_features(
examples=train_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
is_training=True,
languages=args.train_languages)
if args.local_rank == -1 or torch.distributed.get_rank() == 0:
logger.info(" Saving train features into cached file %s",
cached_train_features_file)
with open(cached_train_features_file, "wb") as writer:
pickle.dump(train_features, writer)
logger.info("***** Running training *****")
logger.info(" Num orig examples = %d", len(train_examples))
logger.info(" Num split examples = %d", len(train_features))
logger.info(" Num labels = %d", len(train_examples[0].label_vocab))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_loss_mask = torch.tensor([f.loss_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_labels = torch.tensor([f.label_ids for f in train_features],
dtype=torch.long)
all_language_ids = torch.tensor(
[f.language_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_loss_mask, all_segment_ids, all_labels,
all_language_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
current_f1 = 0.0
best_f1 = 0.0
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
if n_gpu == 1:
batch = tuple(
t.to(device)
for t in batch) # multi-gpu does scattering it-self
input_ids, input_mask, loss_mask, segment_ids, labels, language_ids = batch
loss, adversarial_loss, adversarial_accuracy = model(
input_ids, segment_ids, input_mask, loss_mask, labels,
language_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used and handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
tensorboard_writer.add_scalar('loss', loss.item())
tensorboard_writer.add_scalar('adversarial_loss',
adversarial_loss.item())
tensorboard_writer.add_scalar('adversarial_accuracy',
adversarial_accuracy)
optimizer_params = optimizer.param_groups[-1]
tensorboard_writer.add_scalar(
'weight_decay', optimizer_params["weight_decay"])
tensorboard_writer.add_scalar('learning_rate',
optimizer_params["lr"])
optimizer.step()
optimizer.zero_grad()
global_step += 1
if args.evaluate_each_epoch:
precision, recall, f1 = evaluate_model(model)
tensorboard_writer.add_scalar('precision', precision)
tensorboard_writer.add_scalar('recall', recall)
tensorboard_writer.add_scalar('f1', f1)
if args.early_stopping and epoch > 0:
if f1 < current_f1:
logger.info(
"Stopping early because {} F1 < {} F1".format(
f1, current_f1))
break
if f1 > best_f1:
logger.info("Saving model ...")
model_to_save = model.module if hasattr(
model,
'module') else model # Only save the model it-self
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir,
CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
best_f1 = f1
current_f1 = f1
if not args.evaluate_each_epoch:
logger.info("Saving model ...")
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
del model
if args.do_predict and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = AdversarialBertForNER(config,
num_labels=len(
eval_examples[0].label_vocab),
num_languages=2)
model.load_state_dict(torch.load(output_model_file))
model.to(device)
evaluate_model(model)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
RawResult = collections.namedtuple("RawResult",
["unique_id", "start_logits", "end_logits"])
# para_file = "../Input_file.txt"
para_file = "/content/drive/My Drive/train-v2.0.json" # TODO: use proper file path
model_path = "/content/drive/My Drive/pytorch_model.bin" # TODO: use proper file path
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True)
### Loading Pretrained model for QnA
config = BertConfig("../Results/bert_config.json")
model = BertForQuestionAnswering(config)
model.load_state_dict(torch.load(model_path, map_location='cpu'))
# model = BertForQuestionAnswering.from_pretrained('bert-base-uncased')
model.to(device)
print()
### initializing the autoencoder
hidden_size = 384
encoder1 = EncoderRNN(384, config.hidden_size, hidden_size).to(device)
decoder1 = DecoderRNN(384, config.hidden_size, hidden_size).to(device)
encoder_optimizer = optim.Adam(encoder1.parameters())
decoder_optimizer = optim.Adam(decoder1.parameters())
criterion = nn.MSELoss()
import torch
from pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification
from sentence_fluency.processing import MAX_LENGTH
from utilss.utils import features_translation, tokenizer
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
SOURCE_PATH = '/home/gump/Software/pycharm-2018.1.6/projects/bert-for-classificaion/' \
'sentence_fluency/data/'
CONFIG_FILE = SOURCE_PATH+'model/config_7.json'
MODEL_FILE = SOURCE_PATH+'model/model_7.bin'
# load model
config = BertConfig(CONFIG_FILE)
model = BertForSequenceClassification(config, num_labels=2)
model.load_state_dict(torch.load(MODEL_FILE))
if torch.cuda.is_available():
model.cuda()
model.eval()
tokenizer = tokenizer()
def predict(sentence):
token_a = tokenizer.tokenize(sentence)
token_text = ['[CLS]'] + token_a + ['[SEP]']
tokens_ids = tokenizer.convert_tokens_to_ids(token_text)
def predict(self, text: str, ok=False):
input_ids, input_mask, segment_ids, valid_ids, tokens = self.preprocess(
text)
input_ids = torch.tensor([input_ids], dtype=torch.long)
input_mask = torch.tensor([input_mask], dtype=torch.long)
segment_ids = torch.tensor([segment_ids], dtype=torch.long)
valid_ids = torch.tensor([valid_ids], dtype=torch.long)
model_dir = 'out/'
model_config = "model_config.json"
model_config = os.path.join(model_dir, model_config)
model_config = json.load(open(model_config))
output_config_file = os.path.join(model_dir, CONFIG_NAME)
output_model_file = os.path.join(model_dir, WEIGHTS_NAME)
config = BertConfig(output_config_file)
model = BertNer(config, num_labels=model_config["num_labels"])
bert_embed, _ = model.bert(input_ids,
token_type_ids=None,
attention_mask=None,
output_all_encoded_layers=False)
#print(tokens)
#print(valid_ids)
#print(input_ids)
#print(input_mask)
with torch.no_grad():
logit = self.model(input_ids, segment_ids, input_mask, valid_ids)
logits = F.softmax(logit, dim=2)
logits_label = torch.argmax(logits, dim=2)
logits_label = logits_label.detach().cpu().numpy().tolist()[0]
#print(logits_label)
# import ipdb; ipdb.set_trace()
logits_confidence = [
values[label].item()
for values, label in zip(logits[0], logits_label)
]
logits = []
pos = 0
for index, mask in enumerate(valid_ids[0]):
if index == 0:
continue
if mask == 1:
logits.append((logits_label[index - pos],
logits_confidence[index - pos]))
else:
pos += 1
logits.pop()
#print(logits)
labels = [(self.label_map[label], confidence)
for label, confidence in logits]
#print(labels)
tags = [self.label_map[label] for label, _ in logits]
words = text.split()
#print(tags)
if ok:
classifier = load_model("classifier.h5")
tensor_idx = 0
n_count = 0
embedding = []
for i in range(len(tokens)):
if (valid_ids[0][i] != 0) and (tokens[i] != '[CLS]') and (
tokens[i] != '[SEP]'):
tensor_idx += 1
if tags[tensor_idx - 1] != 'O':
n_count += 1
if n_count == 1:
embedding = [
bert_embed[0, tensor_idx, :].tolist() +
logit[0, tensor_idx, 1:9].tolist()
]
else:
embedding.append(
bert_embed[0, tensor_idx, :].tolist() +
logit[0, tensor_idx, 1:9].tolist())
# print(len(embedding))
#classifier predictions
embedding = np.array(embedding).reshape(-1, 776, 1)
predictions = classifier.predict(embedding)
y_pred = [np.argmax(predictions[i]) for i in range(len(embedding))]
y_pred = [idx2tag[i] for i in y_pred]
tags1 = [self.label_map[label] for label, _ in logits]
j = 0
for i in range(len(tags)):
if tags[i] != 'O' and tags[i][0] != 'I':
tags[i] = "B-" + y_pred[j]
j += 1
elif tags[i][0] == 'I':
tags[i] = "I-" + tags[i - 1][2:]
j += 1
assert len(labels) == len(words)
if ok == True:
print("bert output :\n{}".format(tags1))
output = [(word, {"tag": label}) for word, label in zip(words, tags)]
return output, logit
params = json.loads(f.read())
# loading token & label vocab
token_vocab_path = params['filepath'].get('token_vocab')
label_vocab_path = params['filepath'].get('label_vocab')
with open(token_vocab_path, 'rb') as f:
token_vocab = pickle.load(f)
with open(label_vocab_path, 'rb') as f:
label_vocab = pickle.load(f)
# loading trained model
save_path = params['filepath'].get('ckpt')
# save_path = 'tokenize.pth'
ckpt = torch.load(save_path)
config = BertConfig('bert/bert_config.json')
model = BertTagger(config=config, num_labels=len(label_vocab.token_to_idx), vocab=token_vocab)
model.load_state_dict(ckpt['model_state_dict'])
# loading datasets
batch_size = params['training'].get('batch_size')
train_path = params['filepath'].get('train')
val_path = params['filepath'].get('val')
test_path = params['filepath'].get('test')
train_data = Corpus(train_path, token_vocab.to_indices, label_vocab.to_indices)
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=False, num_workers=16,
drop_last=True, collate_fn=batchify)
val_data = Corpus(val_path, token_vocab.to_indices, label_vocab.to_indices)
val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=False, num_workers=16,
drop_last=True, collate_fn=batchify)
def main(config, model_times, myProcessor):
if not os.path.exists(config.output_dir + model_times):
os.makedirs(config.output_dir + model_times)
if not os.path.exists(config.cache_dir + model_times):
os.makedirs(config.cache_dir + model_times)
# Bert 模型输出文件
output_model_file = os.path.join(config.output_dir, model_times, WEIGHTS_NAME)
output_config_file = os.path.join(config.output_dir, model_times,CONFIG_NAME)
# 设备准备
gpu_ids = [int(device_id) for device_id in config.gpu_ids.split()]
device, n_gpu = get_device(gpu_ids[0])
if n_gpu > 1:
n_gpu = len(gpu_ids)
config.train_batch_size = config.train_batch_size // config.gradient_accumulation_steps
# 设定随机种子
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(config.seed)
# 数据准备
processor = myProcessor() # 整个文件的代码只需要改此处即可
tokenizer = BertTokenizer.from_pretrained(
config.bert_vocab_file, do_lower_case=config.do_lower_case) # 分词器选择
label_list = processor.get_labels()
num_labels = len(label_list)
# Train and dev
if config.do_train:
train_dataloader, train_examples_len = load_data(
config.data_dir, tokenizer, processor, config.max_seq_length, config.train_batch_size, "train")
dev_dataloader, _ = load_data(
config.data_dir, tokenizer, processor, config.max_seq_length, config.dev_batch_size, "dev")
num_train_optimization_steps = int(
train_examples_len / config.train_batch_size / config.gradient_accumulation_steps) * config.num_train_epochs
# 模型准备
print("model name is {}".format(config.model_name))
if config.model_name == "BertOrigin":
from BertOrigin.BertOrigin import BertOrigin
model = BertOrigin.from_pretrained(
config.bert_model_dir, cache_dir=config.cache_dir, num_labels=num_labels)
elif config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNN.from_pretrained(
config.bert_model_dir, cache_dir=config.cache_dir, num_labels=num_labels,
n_filters=config.filter_num, filter_sizes=filter_sizes)
elif config.model_name == "BertATT":
from BertATT.BertATT import BertATT
model = BertATT.from_pretrained(
config.bert_model_dir, cache_dir=config.cache_dir, num_labels=num_labels)
elif config.model_name == "BertRCNN":
from BertRCNN.BertRCNN import BertRCNN
model = BertRCNN.from_pretrained(
config.bert_model_dir, cache_dir=config.cache_dir, num_labels=num_labels, rnn_hidden_size=config.hidden_size, num_layers=config.num_layers, bidirectional=config.bidirectional, dropout=config.dropout)
elif config.model_name == "BertCNNPlus":
from BertCNNPlus.BertCNNPlus import BertCNNPlus
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNNPlus.from_pretrained(
config.bert_model_dir, cache_dir=config.cache_dir, num_labels=num_labels,
n_filters=config.filter_num, filter_sizes=filter_sizes)
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model,device_ids=gpu_ids)
""" 优化器准备 """
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(
nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(
nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=config.warmup_proportion,
t_total=num_train_optimization_steps)
""" 损失函数准备 """
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
train(config.num_train_epochs, n_gpu, model, train_dataloader, dev_dataloader, optimizer,
criterion, config.gradient_accumulation_steps, device, label_list, output_model_file, output_config_file, config.log_dir, config.print_step, config.early_stop)
""" Test """
# test 数据
test_dataloader, _ = load_data(
config.data_dir, tokenizer, processor, config.max_seq_length, config.test_batch_size, "test")
# 加载模型
bert_config = BertConfig(output_config_file)
if config.model_name == "BertOrigin":
from BertOrigin.BertOrigin import BertOrigin
model = BertOrigin(bert_config, num_labels=num_labels)
elif config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNN(bert_config, num_labels=num_labels,
n_filters=config.filter_num, filter_sizes=filter_sizes)
elif config.model_name == "BertATT":
from BertATT.BertATT import BertATT
model = BertATT(bert_config, num_labels=num_labels)
elif config.model_name == "BertRCNN":
from BertRCNN.BertRCNN import BertRCNN
model = BertRCNN(bert_config, num_labels, config.hidden_size, config.num_layers, config.bidirectional, config.dropout)
elif config.model_name == "BertCNNPlus":
from BertCNNPlus.BertCNNPlus import BertCNNPlus
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNNPlus(bert_config, num_labels=num_labels,
n_filters=config.filter_num, filter_sizes=filter_sizes)
model.load_state_dict(torch.load(output_model_file))
model.to(device)
# 损失函数准备
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
# test the model
test_loss, test_acc, test_report, test_auc = evaluate(
model, test_dataloader, criterion, device, label_list)
print("-------------- Test -------------")
print(f'\t Loss: {test_loss: .3f} | Acc: {test_acc*100: .3f} % | AUC:{test_auc}')
for label in label_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'], test_report[label]['recall'], test_report[label]['f1-score']))
print_list = ['macro avg', 'weighted avg']
for label in print_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'], test_report[label]['recall'], test_report[label]['f1-score']))
def main():
logger.info('Launching the MT-DNN training')
opt = vars(args)
# update data dir
opt['data_dir'] = data_dir
batch_size = args.batch_size
# tensorboard
tensorboard = None
if args.tensorboard:
args.tensorboard_logdir = os.path.join(args.output_dir,
args.tensorboard_logdir)
tensorboard = SummaryWriter(log_dir=args.tensorboard_logdir)
json_logfile = os.path.join(args.output_dir, "runtime_log.json")
tasks = {}
tasks_class = {}
nclass_list = []
decoder_opts = []
task_types = []
dropout_list = []
loss_types = []
kd_loss_types = []
train_datasets = []
for dataset in args.train_datasets:
prefix = dataset.split('_')[0]
if prefix in tasks: continue
assert prefix in task_defs.n_class_map
assert prefix in task_defs.data_type_map
data_type = task_defs.data_type_map[prefix]
nclass = task_defs.n_class_map[prefix]
task_id = len(tasks)
if args.mtl_opt > 0:
task_id = tasks_class[nclass] if nclass in tasks_class else len(
tasks_class)
task_type = task_defs.task_type_map[prefix]
dopt = generate_decoder_opt(task_defs.enable_san_map[prefix],
opt['answer_opt'])
if task_id < len(decoder_opts):
decoder_opts[task_id] = min(decoder_opts[task_id], dopt)
else:
decoder_opts.append(dopt)
task_types.append(task_type)
loss_types.append(task_defs.loss_map[prefix])
kd_loss_types.append(task_defs.kd_loss_map[prefix])
if prefix not in tasks:
tasks[prefix] = len(tasks)
if args.mtl_opt < 1: nclass_list.append(nclass)
if (nclass not in tasks_class):
tasks_class[nclass] = len(tasks_class)
if args.mtl_opt > 0: nclass_list.append(nclass)
dropout_p = task_defs.dropout_p_map.get(prefix, args.dropout_p)
dropout_list.append(dropout_p)
train_path = os.path.join(data_dir, '{}_train.json'.format(dataset))
logger.info('Loading {} as task {}'.format(train_path, task_id))
train_data_set = SingleTaskDataset(train_path,
True,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
train_datasets.append(train_data_set)
train_collater = Collater(dropout_w=args.dropout_w,
encoder_type=encoder_type)
multi_task_train_dataset = MultiTaskDataset(train_datasets)
# MTSampler = SAMPLERS[args.sampler]
n_tasks = len(tasks)
dataset_sizes = [len(dataset) for dataset in train_datasets]
if "random" in args.controller:
controller = CONTROLLERS[args.controller](
n_task=n_tasks,
dataset_names=args.train_datasets,
dataset_sizes=dataset_sizes,
batch_size=args.batch_size,
rebatch_size=args.batch_size_train,
tensorboard=tensorboard,
log_filename=json_logfile)
else:
controller = CONTROLLERS[args.controller](
n_task=n_tasks,
phi=args.phi,
K=args.concurrent_cnt,
dataset_names=args.train_datasets,
dataset_sizes=dataset_sizes,
max_cnt=args.max_queue_cnt,
batch_size=args.batch_size,
rebatch_size=args.batch_size_train,
tensorboard=tensorboard,
log_filename=json_logfile)
multi_task_batch_sampler = ACLSampler(train_datasets,
args.batch_size,
controller=controller)
# controller.max_step = len(multi_task_batch_sampler)
multi_task_train_data = DataLoader(multi_task_train_dataset,
batch_sampler=multi_task_batch_sampler,
collate_fn=train_collater.collate_fn,
pin_memory=args.cuda)
opt['answer_opt'] = decoder_opts
opt['task_types'] = task_types
opt['tasks_dropout_p'] = dropout_list
opt['loss_types'] = loss_types
opt['kd_loss_types'] = kd_loss_types
args.label_size = ','.join([str(l) for l in nclass_list])
logger.info(args.label_size)
dev_data_list = []
test_data_list = []
test_collater = Collater(is_train=False, encoder_type=encoder_type)
for dataset in args.test_datasets:
prefix = dataset.split('_')[0]
task_id = tasks_class[
task_defs.
n_class_map[prefix]] if args.mtl_opt > 0 else tasks[prefix]
task_type = task_defs.task_type_map[prefix]
pw_task = False
if task_type == TaskType.Ranking:
pw_task = True
assert prefix in task_defs.data_type_map
data_type = task_defs.data_type_map[prefix]
dev_path = os.path.join(data_dir, '{}_dev.json'.format(dataset))
dev_data = None
if os.path.exists(dev_path):
dev_data_set = SingleTaskDataset(dev_path,
False,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
dev_data = DataLoader(dev_data_set,
batch_size=args.batch_size_eval,
collate_fn=test_collater.collate_fn,
pin_memory=args.cuda)
dev_data_list.append(dev_data)
test_path = os.path.join(data_dir, '{}_test.json'.format(dataset))
test_data = None
if os.path.exists(test_path):
test_data_set = SingleTaskDataset(test_path,
False,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
test_data = DataLoader(test_data_set,
batch_size=args.batch_size_eval,
collate_fn=test_collater.collate_fn,
pin_memory=args.cuda)
test_data_list.append(test_data)
logger.info('#' * 20)
logger.info(opt)
logger.info('#' * 20)
# div number of grad accumulation.
num_all_batches = args.epochs * len(
multi_task_train_data) // args.grad_accumulation_step
logger.info('############# Gradient Accumulation Info #############')
logger.info('number of step: {}'.format(args.epochs *
len(multi_task_train_data)))
logger.info('number of grad grad_accumulation step: {}'.format(
args.grad_accumulation_step))
logger.info('adjusted number of step: {}'.format(num_all_batches))
logger.info('############# Gradient Accumulation Info #############')
bert_model_path = args.init_checkpoint
state_dict = None
if encoder_type == EncoderModelType.BERT:
if os.path.exists(bert_model_path):
state_dict = torch.load(bert_model_path)
config = state_dict['config']
config['attention_probs_dropout_prob'] = args.bert_dropout_p
config['hidden_dropout_prob'] = args.bert_dropout_p
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
else:
logger.error('#' * 20)
logger.error(
'Could not find the init model!\n The parameters will be initialized randomly!'
)
logger.error('#' * 20)
config = BertConfig(vocab_size_or_config_json_file=30522).to_dict()
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
elif encoder_type == EncoderModelType.ROBERTA:
bert_model_path = '{}/model.pt'.format(bert_model_path)
if os.path.exists(bert_model_path):
new_state_dict = {}
state_dict = torch.load(bert_model_path)
for key, val in state_dict['model'].items():
if key.startswith('decoder.sentence_encoder'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
elif key.startswith('classification_heads'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
state_dict = {'state': new_state_dict}
# add score history
score_history = [[] for _ in range(len(args.test_datasets))]
total_scores = []
model = MTDNNModel(opt,
state_dict=state_dict,
num_train_step=num_all_batches)
if args.resume and args.model_ckpt:
logger.info('loading model from {}'.format(args.model_ckpt))
model.load(args.model_ckpt)
#### model meta str
headline = '############# Model Arch of MT-DNN #############'
### print network
logger.info('\n{}\n{}\n'.format(headline, model.network))
# dump config
config_file = os.path.join(output_dir, 'config.json')
with open(config_file, 'w', encoding='utf-8') as writer:
writer.write('{}\n'.format(json.dumps(opt)))
writer.write('\n{}\n{}\n'.format(headline, model.network))
logger.info("Total number of params: {}".format(model.total_param))
for epoch in range(0, args.epochs):
logger.warning('At epoch {0}/{1}'.format(epoch + 1, args.epochs))
start = datetime.now()
total_len = len(controller)
controller.set_epoch(epoch)
for i, (batch_meta, batch_data) in enumerate(multi_task_train_data):
batch_meta, batch_data = Collater.patch_data(
args.cuda, batch_meta, batch_data)
task_id = batch_meta['task_id']
loss = model.calculate_loss(batch_meta, batch_data)
controller.insert(task_id, (batch_meta, batch_data), loss.item())
if i % args.log_per_updates == 0:
ramaining_time = str(
(datetime.now() - start) / (controller.cur_step + 1) *
(total_len - controller.cur_step - 1)).split('.')[0]
logger.info("Epoch {0} Progress {1} / {2} ({3:.2%})".format(
epoch + 1, controller.cur_step, total_len,
controller.cur_step * 1.0 / total_len))
# logger.info("Progress {0} / {1} ({2:.2f}%)".format(i, total_len, i*100.0/total_len))
logger.info(
'Task [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]'
.format(task_id, model.updates, model.train_loss.avg,
ramaining_time))
summary_str = controller.summary()
for line in summary_str.split("\n"):
logger.info(line)
# avg_loss, out_loss, loss_change, min_loss, min_out_loss = controller.get_loss()
# logger.info('List of loss {}'.format(",".join(avg_loss)))
# logger.info('List of out_loss {}'.format(",".join(out_loss)))
# logger.info('List of loss_change {}'.format(",".join(loss_change)))
# logger.info('List of min_loss {}'.format(",".join(min_loss)))
# logger.info('List of min_out_loss {}'.format(",".join(min_out_loss)))
# chosen = [ "%s:%.3f "%(k,v) for k, v in controller.scaled_dict.items()]
# logger.info('List of Scaled Choosen time {}'.format(",".join(chosen)))
if args.tensorboard:
tensorboard.add_scalar('train/loss',
model.train_loss.avg,
global_step=model.updates)
controller.step(model=model)
if args.save_per_updates_on and (
(model.local_updates) %
(args.save_per_updates * args.grad_accumulation_step) == 0):
model_file = os.path.join(
output_dir, 'model_{}_{}.pt'.format(epoch, model.updates))
logger.info('Saving mt-dnn model to {}'.format(model_file))
model.save(model_file)
total_average_score = 0.0
scoring_cnt = 0
score_dict = dict()
scoring_datasets = "cola,sst,mrpc,stsb,qqp,mnli,qnli,rte,wnli".split(
",")
logger.info('Start Testing')
for idx, dataset in enumerate(args.test_datasets):
prefix = dataset.split('_')[0]
label_dict = task_defs.global_map.get(prefix, None)
dev_data = dev_data_list[idx]
if dev_data is not None:
with torch.no_grad():
dev_metrics, dev_predictions, scores, golds, dev_ids = eval_model(
model,
dev_data,
metric_meta=task_defs.metric_meta_map[prefix],
use_cuda=args.cuda,
label_mapper=label_dict,
task_type=task_defs.task_type_map[prefix])
task_score = 0.0
for key, val in dev_metrics.items():
if args.tensorboard:
tensorboard.add_scalar('dev/{}/{}'.format(
dataset, key),
val,
global_step=epoch)
if isinstance(val, str):
logger.warning(
'Task {0} -- epoch {1} -- Dev {2}:\n {3}'.format(
dataset, epoch + 1, key, val))
else:
logger.warning(
'Task {0} -- epoch {1} -- Dev {2}: {3:.2f}'.format(
dataset, epoch + 1, key, val))
task_score += val
if len(dev_metrics) > 1:
task_score /= len(dev_metrics)
logger.warning(
'Task {0} -- epoch {1} -- Dev {2}: {3:.2f}'.format(
dataset, epoch + 1, "Average", task_score))
if prefix in scoring_datasets:
scoring_cnt += 1
if prefix not in score_dict:
score_dict[prefix] = task_score
else:
score_dict[prefix] = (score_dict[prefix] +
task_score) / 2
total_average_score += task_score
score_history[idx].append("%.2f" % task_score)
logger.warning('Task {0} -- epoch {1} -- Dev {2}: {3}'.format(
dataset, epoch + 1, "History", score_history[idx]))
score_file = os.path.join(
output_dir, '{}_dev_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': dev_metrics,
'predictions': dev_predictions,
'uids': dev_ids,
'scores': scores
}
dump(score_file, results)
if args.glue_format_on:
from experiments.glue.glue_utils import submit
official_score_file = os.path.join(
output_dir,
'{}_dev_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
# test eval
test_data = test_data_list[idx]
if test_data is not None:
with torch.no_grad():
test_metrics, test_predictions, scores, golds, test_ids = eval_model(
model,
test_data,
metric_meta=task_defs.metric_meta_map[prefix],
use_cuda=args.cuda,
with_label=False,
label_mapper=label_dict,
task_type=task_defs.task_type_map[prefix])
score_file = os.path.join(
output_dir,
'{}_test_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': test_metrics,
'predictions': test_predictions,
'uids': test_ids,
'scores': scores
}
dump(score_file, results)
if args.glue_format_on:
from experiments.glue.glue_utils import submit
official_score_file = os.path.join(
output_dir,
'{}_test_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
logger.info('[new test scores saved.]')
scoreing_cnt = len(score_dict)
if scoreing_cnt > 0:
mean_value = np.mean([v for k, v in score_dict.items()])
logger.warning(
'Epoch {0} -- Dev {1} Tasks, Average Score : {2:.3f}'.format(
epoch + 1, scoring_cnt, mean_value))
score_dict['avg'] = mean_value
total_scores.append(score_dict)
model_file = os.path.join(output_dir, 'model_{}.pt'.format(epoch))
model.save(model_file)
for i, total_score in enumerate(total_scores):
logger.info(total_score)
if args.tensorboard:
tensorboard.close()
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .csv files (or other data files) for the task."
)
parser.add_argument(
"--output_sentvec_file",
default=None,
type=str,
required=True,
help="The output file of extracted embedding files of sentences.")
parser.add_argument(
"--data_split_to_extract",
default=None,
type=str,
required=True,
help="The output file of extracted embedding files of sentences.")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument("--epoch_id",
default=0,
type=int,
help="Epoch id to extract.")
parser.add_argument(
"--save_model_name",
default="model",
type=str,
required=True,
help=
"The output model name where the model checkpoints will be written.")
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--with_dev",
action='store_true',
help="Whether to run training with dev.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
action='store_true',
help="Whether to run test on the test set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--layer_id",
default=-1,
type=int,
help="Output Layer Id")
parser.add_argument("--mlp_hidden_dim",
default=64,
type=int,
help="mlp_hidden_dim.")
parser.add_argument("--mlp_dropout",
default=0.1,
type=float,
help="hidden drop out")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--patience',
type=int,
default=5,
help="early stop epoch nums on dev")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
print("torch.cuda.is_available()", torch.cuda.is_available())
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
print(
"WARNING: Output directory ({}) already exists and is not empty.".
format(args.output_dir))
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = read_csqa_examples(
os.path.join(args.data_dir, 'train_rand_split.jsonl'))
dev_examples = read_csqa_examples(
os.path.join(args.data_dir, 'dev_rand_split.jsonl'))
print(len(train_examples))
if args.with_dev:
train_examples += dev_examples
print(len(train_examples))
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
model = BertForMultipleChoiceExtraction.from_pretrained(
args.bert_model,
cache_dir=os.path.join(PYTORCH_PRETRAINED_BERT_CACHE,
'distributed_{}'.format(args.local_rank)),
num_choices=5,
mlp_hidden_dim=args.mlp_hidden_dim,
mlp_dropout=args.mlp_dropout)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
# hack to remove pooler, which is not used
# thus it produce None grad that break apex
param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Load a trained model and config that you have fine-tuned
output_model_file = os.path.join(
args.output_dir,
args.save_model_name + ".bin.%d" % (args.epoch_id))
output_config_file = os.path.join(args.output_dir,
args.save_model_name + ".config")
config = BertConfig(output_config_file)
model = BertForMultipleChoiceExtraction(
config,
num_choices=5,
mlp_hidden_dim=args.mlp_hidden_dim,
mlp_dropout=args.mlp_dropout)
model.load_state_dict(torch.load(output_model_file))
model.to(device)
# to extract dev_rand_split.jsonl 'dev_rand_split.jsonl'
eval_examples = read_csqa_examples(
os.path.join(args.data_dir, args.data_split_to_extract))
eval_features = convert_examples_to_features(eval_examples, tokenizer,
args.max_seq_length, True)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'),
dtype=torch.long)
all_input_mask = torch.tensor(select_field(eval_features,
'input_mask'),
dtype=torch.long)
all_segment_ids = torch.tensor(select_field(eval_features,
'segment_ids'),
dtype=torch.long)
all_label = torch.tensor([f.label for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
pooled_sent_vecs = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Iteration"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
# tmp_eval_loss, pooled_output = model(input_ids, segment_ids, input_mask, label_ids)
logits, pooled_output = model(input_ids,
segment_ids,
input_mask,
layer_id=args.layer_id)
pooled_sent_vecs.append(pooled_output)
# print(pooled_output.size())
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
pooled_sent_vecs = torch.cat(pooled_sent_vecs, dim=0)
print(pooled_sent_vecs.size())
output_numpy = pooled_sent_vecs.to('cpu').numpy()
print(output_numpy.shape)
np.save(args.output_sentvec_file + ".%d" % (args.layer_id),
output_numpy)
eval_accuracy = eval_accuracy / nb_eval_examples
result = {'eval_accuracy': eval_accuracy}
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=330,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
action='store_true',
help="Whether to run test on the test set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=32,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
output_modes = {
"cola": "classification",
"mnli": "classification",
"mrpc": "classification",
"sst-2": "classification",
"sts-b": "regression",
"qqp": "classification",
"qnli": "classification",
"rte": "classification",
"wnli": "classification",
}
# '''. prepare data
args.stance_dir = args.data_dir
args.ir_dir = "/home/Vachel/github/pytorch-pretrained-BERT_old/data/source"
# tokenize = lambda x: [w for w in x]
# text_field = data.Field(sequential=True, tokenize=tokenize, lower=True, batch_first = True)
# label_field = data.Field(sequential=False, use_vocab=False)
# train_stance_iter, dev_stance_iter, test_stance_iter = mydatasets_cn.stance_datase, label_field, args)
# train_iter, dev_iter = mydatasets_cn.ir_datase, args)
label_list = ["0", "1"]
num_labels = len(label_list)
stance_dataset = Stance_Dataset(args, "train.tsv", "dev.tsv", "test.tsv")
ir_dataset = IR_Dataset(args, "train.tsv", "dev.tsv")
# '''
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError(
"At least one of do_train` or `do_eval` or `do_test` must be True."
)
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
output_mode = output_modes[task_name]
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
num_train_optimization_steps = None
if args.do_train:
# '''
num_train_optimization_steps = int(
len(stance_dataset.train_data) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
# '''
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(stance_dataset.train_data))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
# for step, batch_stance in enumerate(tqdm(train_stance_iter, desc="Iteration")):
train_stance_iter, dev_stance_iter, test_stance_iter = stance_dataset.Iterator(
)
train_iter, dev_iter = ir_dataset.Iterator()
for step, (batch, batch_stance) in enumerate(
tqdm(zip(train_iter, train_stance_iter),
desc="Iteration")):
input_ids, input_mask, segment_ids, label_ids = to_bert_input_related(
args, batch, label_list, tokenizer, output_mode)
input_ids_stance, input_mask_stance, segment_ids_stance, label_ids_stance = to_bert_input_stance(
args, batch_stance, label_list, tokenizer, output_mode)
input_ids = torch.cat((input_ids, input_ids_stance), dim=0)
input_mask = torch.cat((input_mask, input_mask_stance), dim=0)
segment_ids = torch.cat((segment_ids, segment_ids_stance),
dim=0)
label_ids = torch.cat((label_ids, label_ids_stance), dim=0)
# input_ids, input_mask, segment_ids, label_ids = to_bert_input_stance(args, batch_stance, label_list, tokenizer, output_mode)
input_ids, input_mask, segment_ids, label_ids = input_ids.to(
device), input_mask.to(device), segment_ids.to(
device), label_ids.to(device)
# define a new function to compute loss values for both output_modes
logits = model(input_ids, segment_ids, input_mask, labels=None)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# eval
logger.info("***** Running evaluation *****")
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
all_label_ids = []
# for step, batch_stance in enumerate(tqdm(dev_stance_iter, desc="Evaluating")):
for step, (batch, batch_stance) in enumerate(
tqdm(zip(dev_iter, dev_stance_iter), desc="Evaluating")):
input_ids, input_mask, segment_ids, label_ids = to_bert_input_related(
args, batch, label_list, tokenizer, output_mode)
input_ids_stance, input_mask_stance, segment_ids_stance, label_ids_stance = to_bert_input_stance(
args, batch_stance, label_list, tokenizer, output_mode)
input_ids = torch.cat((input_ids, input_ids_stance), dim=0)
input_mask = torch.cat((input_mask, input_mask_stance), dim=0)
segment_ids = torch.cat((segment_ids, segment_ids_stance),
dim=0)
label_ids = torch.cat((label_ids, label_ids_stance), dim=0)
# input_ids, input_mask, segment_ids, label_ids = to_bert_input_stance(args, batch_stance, label_list, tokenizer, output_mode)
all_label_ids += list(label_ids.numpy())
input_ids, input_mask, segment_ids, label_ids = input_ids.to(
device), input_mask.to(device), segment_ids.to(
device), label_ids.to(device)
with torch.no_grad():
logits = model(input_ids,
segment_ids,
input_mask,
labels=None)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_eval_loss = loss_fct(logits.view(-1),
label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
exp_x = np.exp(preds)
softmax_x = exp_x / np.mat(np.sum(exp_x, axis=1)).T
# np.save("eval_logits.npy", softmax_x)
# np.save("eval_labels.npy", all_label_ids)
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
elif output_mode == "regression":
preds = np.squeeze(preds)
result = compute_metrics(task_name, preds, all_label_ids)
loss = tr_loss / nb_tr_steps if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir,
"eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_test and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
logger.info("***** Running testing *****")
# Load a trained model and config that you have fine-tuned
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
if args.fp16:
model.half()
model.to(device)
model.eval()
train_stance_iter, dev_stance_iter, test_stance_iter = stance_dataset.Iterator(
)
for i, batch in enumerate(tqdm(test_stance_iter, desc="Evaluating")):
test_src_batch, test_tgt_batch = batch
input_ids, input_mask, segment_ids = to_bert_input_test(
args, test_src_batch, test_tgt_batch, label_list, tokenizer,
output_mode)
input_ids, input_mask, segment_ids = input_ids.to(
device), input_mask.to(device), segment_ids.to(device)
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
if i == 0:
pred_poss = list(
F.softmax(logits, dim=1).detach().cpu().numpy())
else:
pred_poss += list(
F.softmax(logits, dim=1).detach().cpu().numpy())
pred_poss = np.array(pred_poss)
np.save("preds_stance.npy", pred_poss)
print(pred_poss.shape)
def main():
logger.info('Launching the MT-DNN training')
opt = vars(args)
# update data dir
opt['data_dir'] = data_dir
batch_size = args.batch_size
tasks = {}
tasks_class = {}
nclass_list = []
decoder_opts = []
task_types = []
dropout_list = []
loss_types = []
kd_loss_types = []
train_datasets = []
for dataset in args.train_datasets:
prefix = dataset.split('_')[0]
if prefix in tasks: continue
assert prefix in task_defs.n_class_map
assert prefix in task_defs.data_type_map
data_type = task_defs.data_type_map[prefix]
nclass = task_defs.n_class_map[prefix]
task_id = len(tasks)
if args.mtl_opt > 0:
task_id = tasks_class[nclass] if nclass in tasks_class else len(
tasks_class)
task_type = task_defs.task_type_map[prefix]
dopt = generate_decoder_opt(task_defs.enable_san_map[prefix],
opt['answer_opt'])
if task_id < len(decoder_opts):
decoder_opts[task_id] = min(decoder_opts[task_id], dopt)
else:
decoder_opts.append(dopt)
task_types.append(task_type)
loss_types.append(task_defs.loss_map[prefix])
kd_loss_types.append(task_defs.kd_loss_map[prefix])
if prefix not in tasks:
tasks[prefix] = len(tasks)
if args.mtl_opt < 1: nclass_list.append(nclass)
if (nclass not in tasks_class):
tasks_class[nclass] = len(tasks_class)
if args.mtl_opt > 0: nclass_list.append(nclass)
dropout_p = task_defs.dropout_p_map.get(prefix, args.dropout_p)
dropout_list.append(dropout_p)
train_path = os.path.join(data_dir, '{}_train.json'.format(dataset))
logger.info('Loading {} as task {}'.format(train_path, task_id))
train_data_set = SingleTaskDataset(train_path,
True,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
train_datasets.append(train_data_set)
train_collater = Collater(dropout_w=args.dropout_w,
encoder_type=encoder_type)
multi_task_train_dataset = MultiTaskDataset(train_datasets)
multi_task_batch_sampler = MultiTaskBatchSampler(train_datasets,
args.batch_size,
args.mix_opt, args.ratio)
multi_task_train_data = DataLoader(multi_task_train_dataset,
batch_sampler=multi_task_batch_sampler,
collate_fn=train_collater.collate_fn,
pin_memory=args.cuda)
opt['answer_opt'] = decoder_opts
opt['task_types'] = task_types
opt['tasks_dropout_p'] = dropout_list
opt['loss_types'] = loss_types
opt['kd_loss_types'] = kd_loss_types
args.label_size = ','.join([str(l) for l in nclass_list])
logger.info(args.label_size)
dev_data_list = []
test_data_list = []
test_collater = Collater(is_train=False, encoder_type=encoder_type)
for dataset in args.test_datasets:
prefix = dataset.split('_')[0]
task_id = tasks_class[
task_defs.
n_class_map[prefix]] if args.mtl_opt > 0 else tasks[prefix]
task_type = task_defs.task_type_map[prefix]
pw_task = False
if task_type == TaskType.Ranking:
pw_task = True
assert prefix in task_defs.data_type_map
data_type = task_defs.data_type_map[prefix]
dev_path = os.path.join(data_dir, '{}_dev.json'.format(dataset))
dev_data = None
if os.path.exists(dev_path):
dev_data_set = SingleTaskDataset(dev_path,
False,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
dev_data = DataLoader(dev_data_set,
batch_size=args.batch_size_eval,
collate_fn=test_collater.collate_fn,
pin_memory=args.cuda)
dev_data_list.append(dev_data)
test_path = os.path.join(data_dir, '{}_test.json'.format(dataset))
test_data = None
if os.path.exists(test_path):
test_data_set = SingleTaskDataset(test_path,
False,
maxlen=args.max_seq_len,
task_id=task_id,
task_type=task_type,
data_type=data_type)
test_data = DataLoader(test_data_set,
batch_size=args.batch_size_eval,
collate_fn=test_collater.collate_fn,
pin_memory=args.cuda)
test_data_list.append(test_data)
logger.info('#' * 20)
logger.info(opt)
logger.info('#' * 20)
# div number of grad accumulation.
num_all_batches = args.epochs * len(
multi_task_train_data) // args.grad_accumulation_step
logger.info('############# Gradient Accumulation Info #############')
logger.info('number of step: {}'.format(args.epochs *
len(multi_task_train_data)))
logger.info('number of grad grad_accumulation step: {}'.format(
args.grad_accumulation_step))
logger.info('adjusted number of step: {}'.format(num_all_batches))
logger.info('############# Gradient Accumulation Info #############')
bert_model_path = args.init_checkpoint
state_dict = None
if encoder_type == EncoderModelType.BERT:
if os.path.exists(bert_model_path):
state_dict = torch.load(bert_model_path)
config = state_dict['config']
config['attention_probs_dropout_prob'] = args.bert_dropout_p
config['hidden_dropout_prob'] = args.bert_dropout_p
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
else:
logger.error('#' * 20)
logger.error(
'Could not find the init model!\n The parameters will be initialized randomly!'
)
logger.error('#' * 20)
config = BertConfig(vocab_size_or_config_json_file=30522).to_dict()
config['multi_gpu_on'] = opt["multi_gpu_on"]
opt.update(config)
elif encoder_type == EncoderModelType.ROBERTA:
bert_model_path = '{}/model.pt'.format(bert_model_path)
if os.path.exists(bert_model_path):
new_state_dict = {}
state_dict = torch.load(bert_model_path)
for key, val in state_dict['model'].items():
if key.startswith('decoder.sentence_encoder'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
elif key.startswith('classification_heads'):
key = 'bert.model.{}'.format(key)
new_state_dict[key] = val
state_dict = {'state': new_state_dict}
model = MTDNNModel(opt,
state_dict=state_dict,
num_train_step=num_all_batches)
if args.resume and args.model_ckpt:
logger.info('loading model from {}'.format(args.model_ckpt))
model.load(args.model_ckpt)
#### model meta str
headline = '############# Model Arch of MT-DNN #############'
### print network
logger.info('\n{}\n{}\n'.format(headline, model.network))
# dump config
config_file = os.path.join(output_dir, 'config.json')
with open(config_file, 'w', encoding='utf-8') as writer:
writer.write('{}\n'.format(json.dumps(opt)))
writer.write('\n{}\n{}\n'.format(headline, model.network))
logger.info("Total number of params: {}".format(model.total_param))
# tensorboard
if args.tensorboard:
args.tensorboard_logdir = os.path.join(args.output_dir,
args.tensorboard_logdir)
tensorboard = SummaryWriter(log_dir=args.tensorboard_logdir)
for epoch in range(0, args.epochs):
logger.warning('At epoch {}'.format(epoch))
start = datetime.now()
for i, (batch_meta, batch_data) in enumerate(multi_task_train_data):
batch_meta, batch_data = Collater.patch_data(
args.cuda, batch_meta, batch_data)
task_id = batch_meta['task_id']
model.update(batch_meta, batch_data)
if (model.local_updates) % (args.log_per_updates *
args.grad_accumulation_step
) == 0 or model.local_updates == 1:
ramaining_time = str(
(datetime.now() - start) / (i + 1) *
(len(multi_task_train_data) - i - 1)).split('.')[0]
logger.info(
'Task [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]'
.format(task_id, model.updates, model.train_loss.avg,
ramaining_time))
if args.tensorboard:
tensorboard.add_scalar('train/loss',
model.train_loss.avg,
global_step=model.updates)
if args.save_per_updates_on and (
(model.local_updates) %
(args.save_per_updates * args.grad_accumulation_step) == 0):
model_file = os.path.join(
output_dir, 'model_{}_{}.pt'.format(epoch, model.updates))
logger.info('Saving mt-dnn model to {}'.format(model_file))
model.save(model_file)
for idx, dataset in enumerate(args.test_datasets):
prefix = dataset.split('_')[0]
label_dict = task_defs.global_map.get(prefix, None)
dev_data = dev_data_list[idx]
if dev_data is not None:
with torch.no_grad():
dev_metrics, dev_predictions, scores, golds, dev_ids = eval_model(
model,
dev_data,
metric_meta=task_defs.metric_meta_map[prefix],
use_cuda=args.cuda,
label_mapper=label_dict,
task_type=task_defs.task_type_map[prefix])
for key, val in dev_metrics.items():
if args.tensorboard:
tensorboard.add_scalar('dev/{}/{}'.format(
dataset, key),
val,
global_step=epoch)
if isinstance(val, str):
logger.warning(
'Task {0} -- epoch {1} -- Dev {2}:\n {3}'.format(
dataset, epoch, key, val))
else:
logger.warning(
'Task {0} -- epoch {1} -- Dev {2}: {3:.3f}'.format(
dataset, epoch, key, val))
score_file = os.path.join(
output_dir, '{}_dev_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': dev_metrics,
'predictions': dev_predictions,
'uids': dev_ids,
'scores': scores
}
dump(score_file, results)
if args.glue_format_on:
from experiments.glue.glue_utils import submit
official_score_file = os.path.join(
output_dir,
'{}_dev_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
# test eval
test_data = test_data_list[idx]
if test_data is not None:
with torch.no_grad():
test_metrics, test_predictions, scores, golds, test_ids = eval_model(
model,
test_data,
metric_meta=task_defs.metric_meta_map[prefix],
use_cuda=args.cuda,
with_label=False,
label_mapper=label_dict,
task_type=task_defs.task_type_map[prefix])
score_file = os.path.join(
output_dir,
'{}_test_scores_{}.json'.format(dataset, epoch))
results = {
'metrics': test_metrics,
'predictions': test_predictions,
'uids': test_ids,
'scores': scores
}
dump(score_file, results)
if args.glue_format_on:
from experiments.glue.glue_utils import submit
official_score_file = os.path.join(
output_dir,
'{}_test_scores_{}.tsv'.format(dataset, epoch))
submit(official_score_file, results, label_dict)
logger.info('[new test scores saved.]')
model_file = os.path.join(output_dir, 'model_{}.pt'.format(epoch))
model.save(model_file)
if args.tensorboard:
tensorboard.close()
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--dataset_name",
default="top300_kl",
type=str,
required=True,
help="The name of dataset to inference (without extention ex) top300_kl)")
parser.add_argument("--model_type",
default="baseline_tfidf",
type=str,
required=True,
help="baseline, baseline_tfidf, ir-v0, ir-v1")
parser.add_argument("--model_path",
default=None,
type=str,
required=True,
help="path to model dir")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="save_path")
## Other parameters
parser.add_argument("--bert_model",
default="bert-base-multilingual-cased",
type=str,
help="Default: bert-base-multilingual-cased"
"Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--model_file",
default="pytorch_model.bin",
type=str,
help="The file of model (.bin), default is pytorhc_model.bin,\n"
"특정 파일이 필요시 이름 설정 필요")
parser.add_argument("--max_seq_length",
default=384,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
processor = IRProcessor()
label_list = processor.get_labels()
num_labels = len(label_list)
print("model:", args.model_type)
if args.model_type == "baseline": # load model (finetuned baseline on IR)
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=False)
config = BertConfig(os.path.join(args.model_path + "bert_config.json"))
model = BertForPreTraining(config)
model.load_state_dict(torch.load(os.path.join(args.model_path, args.model_file)))
elif args.model_type == "baseline_tfidf": # load model (baseline_tfidf)
tokenizer = BertTFIDFTokenizer.from_pretrained(args.bert_model, do_lower_case=False, do_tf_idf=True)
TFIDFconfig = modeling.BertConfig(os.path.join(args.model_path + "bert_config.json"))
model = modeling.BertTFIDFForPreTraining(TFIDFconfig)
model.load_state_dict(torch.load(os.path.join(args.model_path, args.model_file)))
elif args.model_type == "ir-v0": # load model (*-head)
tokenizer = BertTFIDFTokenizer.from_pretrained(args.bert_model, do_lower_case=False, do_tf_idf=True)
head_config = modeling_ir.BertForIRConfig(os.path.join(args.model_path + "bert_config.json"))
model = modeling_ir.BertForIRForPreTraining(head_config)
model.load_state_dict(torch.load(os.path.join(args.model_path, args.model_file)))
elif args.model_type == "ir-v1": # load model (*-head)
tokenizer = BertTFIDFTokenizer.from_pretrained(args.bert_model, do_lower_case=False, do_tf_idf=True)
head_config = modeling_ir_2.BertForIRConfig(os.path.join(args.model_path + "bert_config.json"))
model = modeling_ir_2.BertForIRForPreTraining(head_config)
model.load_state_dict(torch.load(os.path.join(args.model_path, args.model_file)))
if args.fp16:
model.half()
model.to(device)
tfidf_dict = pickle_load(os.path.join(args.data_dir, args.dataset_name + '_tfidf.pkl'))
results_logit = dict()
results_softmax = dict()
eval_set, documents, queries = processor.make_eval_set(args.data_dir, args.dataset_name)
logger.info("***** Running evaluation *****")
logger.info(" Batch size = %d", args.eval_batch_size)
for q_num, query in tqdm(enumerate(queries), total=len(queries), desc="Evaluating"):
# for query in queries[0:1]: # for testing
logger.info(f"Current Query Num : {q_num}")
eval_examples = processor._create_examples(eval_set, query, documents)
# logger.info(" Num examples = %d", len(eval_examples))
if args.model_type == "baseline": # baseline or baseline_finetuned
eval_features = convert_examples_to_features_for_vanilla(
eval_examples, label_list, args.max_seq_length, tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Query"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
_, logits = model(input_ids, segment_ids, input_mask)
# loss_fct = CrossEntropyLoss()
# tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
# eval_loss += tmp_eval_loss.mean().item()
# nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
else: # baseline_tfidf or *-head model
eval_data = LazyDatasetClassifier(eval_examples, label_list, args.max_seq_length, tokenizer, tfidf_dict)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for batch in tqdm(eval_dataloader, desc="Query"):
batch = tuple(t.to(device) for t in batch)
input_ids, input_weights, input_mask, segment_ids, label_ids = batch
with torch.no_grad():
_, logits = model(input_ids, input_weights, segment_ids, input_mask)
# loss_fct = CrossEntropyLoss()
# tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
# eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
# eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
results_softmax[query] = []
for i, pred in enumerate(softmax(preds)): # using softmax
pair = dict()
pair["score"] = pred[1]
pair["doc_id"] = list(documents.keys())[i]
results_softmax[query].append(pair)
results_softmax[query].sort(reverse=True, key=lambda x: x["score"])
ranked_doc_list = []
for doc in results_logit[query]:
ranked_doc_list.append(doc["doc_id"])
results_logit[query] = ranked_doc_list
ranked_doc_list = []
for doc in results_softmax[query]:
ranked_doc_list.append(doc["doc_id"])
results_softmax[query] = ranked_doc_list
save_name2 = args.model_path.split('/')[0] + '_' + args.model_file.split('.')[0] \
+ '_' + args.dataset_name + '_output.json'
path2 = os.path.join(args.output_dir,
save_name2)
with open(path2, 'w', encoding="utf8") as f:
json.dump(results_softmax, f, indent=4, sort_keys=True, ensure_ascii=False)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--model_dir",
default=None,
type=str,
required=True,
help="")
parser.add_argument("--my_config", default=None, type=str, required=True)
parser.add_argument("--feature_path",
default=None,
type=str,
required=True)
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model checkpoints and predictions will be written."
)
## Other parameters
parser.add_argument("--train_pattern",
default=None,
type=str,
help="training data path.")
parser.add_argument("--valid_pattern",
default=None,
type=str,
help="validation data path.")
parser.add_argument("--test_pattern",
default=None,
type=str,
help="test data path.")
parser.add_argument(
"--max_seq_length",
default=512,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded."
)
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
parser.add_argument(
"--max_query_length",
default=64,
type=int,
help=
"The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--report_steps",
default=100,
type=int,
help="report steps when training.")
parser.add_argument("--train_batch_size",
default=4,
type=int,
help="Total batch size for training.")
parser.add_argument("--predict_batch_size",
default=8,
type=int,
help="Total batch size for predictions.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_steps", default=-1, type=int)
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% "
"of training.")
parser.add_argument(
"--n_best_size",
default=20,
type=int,
help=
"The total number of n-best predictions to generate in the nbest_predictions.json "
"output file.")
parser.add_argument(
"--max_answer_length",
default=30,
type=int,
help=
"The maximum length of an answer that can be generated. This is needed because the start "
"and end predictions are not conditioned on one another.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--train_size',
type=int,
default=10000,
help="Use how many train data")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
"--do_lower_case",
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--frame_name', type=str, default='albert-base-v2')
parser.add_argument('--DataName', type=str, default="SST")
args = parser.parse_args()
#print(args)
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
n_gpu = 1
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_pattern:
raise ValueError(
"If `do_train` is True, then `train_pattern` must be specified."
)
if args.do_predict:
if not args.test_pattern:
raise ValueError(
"If `do_predict` is True, then `test_pattern` must be specified."
)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# Prepare model
my_config = Config(args.my_config)
my_config.num_edge_types = sum(EdgePosition.max_edge_types)
my_config.forward_edges = [
EdgeType.TOKEN_TO_SENTENCE, EdgeType.SENTENCE_TO_PARAGRAPH,
EdgeType.PARAGRAPH_TO_DOCUMENT
]
#print(my_config)
if args.do_train:
pretrained_config_file = os.path.join(args.model_dir, CONFIG_NAME)
bert_config = BertConfig(pretrained_config_file)
pretrained_model_file = os.path.join(args.model_dir, WEIGHTS_NAME)
model = NqModel(bert_config=bert_config, my_config=my_config)
#model_dict = model.state_dict()
#pretrained_model_dict = torch.load(pretrained_model_file, map_location=lambda storage, loc: storage)
#pretrained_model_dict = {k: v for k, v in pretrained_model_dict.items() if k in model_dict.keys()}
#model_dict.update(pretrained_model_dict)
#model.load_state_dict(model_dict)
else:
pretrained_config_file = os.path.join(args.model_dir, CONFIG_NAME)
bert_config = BertConfig(pretrained_config_file)
model = NqModel(bert_config=bert_config, my_config=my_config)
pretrained_model_file = os.path.join(args.model_dir, WEIGHTS_NAME)
model.load_state_dict(torch.load(pretrained_model_file))
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
num_train_features = None
num_train_optimization_steps = None
#train_dataset = None
#train_features = None
if args.do_train:
num_train_features = 0
for data_path in glob(args.train_pattern):
train_dataset = NqDataset(args, data_path, is_training=True)
train_features = train_dataset.features
num_train_features += len(train_dataset.features)
num_train_optimization_steps = int(
num_train_features / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
#print([i for i,j in model.named_parameters()])
#exit(0)
# hack to remove pooler, which is not used
# thus it produce None grad that break apex
param_optimizer = [n for n in param_optimizer]
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.011
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.00
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
if args.warmup_steps > 0:
args.warmup_proportion = min(
args.warmup_proportion,
args.warmup_steps / num_train_optimization_steps)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num split examples = %d", num_train_features)
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
model.train()
tr_loss, report_loss = 0.0, 0.0
nb_tr_examples = 0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
for data_path in glob(args.train_pattern):
#logging.info("Reading data from {}.".format(data_path))
if train_dataset == None:
train_dataset = NqDataset(args,
data_path,
is_training=True)
train_features = train_dataset.features
#logging.info("Data Load Done!")
if args.local_rank == -1:
train_sampler = RandomSampler(train_features)
else:
train_sampler = DistributedSampler(train_features)
train_dataloader = DataLoader(train_features,
sampler=train_sampler,
batch_size=args.train_batch_size,
collate_fn=batcher(
device, is_training=True),
num_workers=0)
train_features = train_dataset.features
logging.info("Data ready {} ".format(len(train_features)))
for step, batch in enumerate(train_dataloader):
loss = model(batch.input_ids, batch.input_mask,
batch.segment_ids, batch.st_mask,
batch.st_index,
(batch.edges_src, batch.edges_tgt,
batch.edges_type, batch.edges_pos),
batch.label)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.local_rank != -1:
loss = loss + 0 * sum(
[x.sum() for x in model.parameters()])
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used and handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
tr_loss += loss.item()
nb_tr_examples += 1
if (step + 1) % args.gradient_accumulation_steps == 0 and (
global_step + 1) % args.report_steps == 0 and (
args.local_rank == -1
or torch.distributed.get_rank() == 0):
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
logging.info(
"Epoch={} iter={} lr={:.6f} train_ave_loss={:.6f} ."
.format(
# _, global_step, lr_this_step, tr_loss / nb_tr_examples))
_,
global_step,
lr_this_step,
(tr_loss - report_loss) / args.report_steps))
report_loss = tr_loss
if args.valid_pattern and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
valid_result = eval_model(args, device, model,
args.valid_pattern)
logging.info("valid_result = {}".format(valid_result))
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Save a trained model, configuration and tokenizer
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
bert_config = BertConfig(output_config_file)
model = NqModel(bert_config=bert_config, my_config=my_config)
model.load_state_dict(torch.load(output_model_file))
if args.fp16:
model.half()
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.do_predict and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
test_result = eval_model(args, device, model, args.test_pattern)
logging.info("test_result = {}".format(test_result))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default="",
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default="",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=50,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
action='store_true',
help="Whether to run eval on the test set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=32,
type=int,
help="Total batch size for eval.")
parser.add_argument("--test_batch_size",
default=32,
type=int,
help="Total batch size for test.")
parser.add_argument("--learning_rate",
default=2e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=2.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
processors = {"task_a": taskA_Processor}
output_modes = {"task_a": "classification"}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError(
"At least one of `do_train` or `do_eval` or `do_test` must be True."
)
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples()
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
# define a new function to compute loss values for both output_modes
logits = model(input_ids, segment_ids, input_mask, labels=None)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
config = BertConfig.from_json_file(
os.path.join(args.output_dir, "bert_config.json"))
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(
torch.load(os.path.join(args.output_dir, "pytorch_model.bin")))
model.to(device)
eval_examples = processor.get_dev_examples()
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
result = compute_metrics(task_name, preds, all_label_ids.numpy())
loss = tr_loss / nb_tr_steps if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
result['model_spec'] = args
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_test and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
config = BertConfig.from_json_file(
os.path.join(args.output_dir, "bert_config.json"))
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(
torch.load(os.path.join(args.output_dir, "pytorch_model.bin")))
model.to(device)
if args.task_name == "task_a":
test_examples = processor.get_test_examples()
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Prediction for Test data *****")
logger.info(" Num examples = %d", len(test_examples))
logger.info(" Batch size = %d", args.test_batch_size)
all_guids = [f.guid for f in test_features]
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids)
# Run prediction for full data
test_dataloader = DataLoader(test_data,
batch_size=args.test_batch_size)
model.eval()
ids = all_guids
preds = []
for input_ids, input_mask, segment_ids in tqdm(test_dataloader,
desc="Prediction"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
# create eval loss and other metric required by the task
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
preds = preds[0]
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
df_preds = pd.DataFrame(preds)
if args.task_name == "task_a":
df_preds = df_preds.replace(
[0, 1, 2, 3], ["support", "deny", "query", "comment"])
preds_list = list(df_preds[0])
result = {"subtaskaenglish": {}, "subtaskbenglish": {}}
for i, id in enumerate(ids):
result["subtaskaenglish"][str(id)] = preds_list[i]
# task_b labels to randomly
tmp_1 = get_all_sources_ids('test', 'twitter')
tmp_2 = get_all_sources_ids('test', 'reddit')
task_b_ids = tmp_1 + tmp_2
scores = np.random.rand(len(task_b_ids))
for i, id in enumerate(task_b_ids):
if scores[i] < 0.33:
result["subtaskbenglish"][str(id)] = ["false", scores[i]]
elif scores[i] >= 0.33 and scores[i] <= 0.66:
result["subtaskbenglish"][str(id)] = ["unverified", 0.5]
else:
result["subtaskbenglish"][str(id)] = ["true", scores[i]]
json.dump(
result,
open(
os.path.join(args.output_dir,
'test_result_%s.json' % args.task_name), "w"))
logger.info("***** Prediction for Test data is completed *****")
def main(config, model_times, myProcessor):
if not os.path.exists(config.output_dir + model_times):
os.makedirs(config.output_dir + model_times)
if not os.path.exists(config.cache_dir + model_times):
os.makedirs(config.cache_dir + model_times)
output_model_file = os.path.join(config.output_dir, model_times,
WEIGHTS_NAME) # 模型输出文件
output_config_file = os.path.join(config.output_dir, model_times,
CONFIG_NAME)
gpu_ids = [int(device_id) for device_id in config.gpu_ids.split()]
device, n_gpu = get_device(gpu_ids[0]) # 设备准备
if n_gpu > 1:
n_gpu = len(gpu_ids)
config.train_batch_size = config.train_batch_size // config.gradient_accumulation_steps
""" 设定随机种子 """
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(config.seed)
""" 数据准备 """
processor = myProcessor() # 整个文件的代码只需要改此处即可
tokenizer = BertTokenizer.from_pretrained(
config.bert_vocab_file, do_lower_case=config.do_lower_case) # 分词器选择
label_list = processor.get_labels()
num_labels = len(label_list)
if config.do_train:
train_dataloader, train_examples_len = load_data(
config.data_dir, tokenizer, processor, config.max_seq_length,
config.train_batch_size, "train")
dev_dataloader, _ = load_data(config.data_dir, tokenizer, processor,
config.max_seq_length,
config.dev_batch_size, "dev")
num_train_optimization_steps = int(
train_examples_len / config.train_batch_size /
config.gradient_accumulation_steps) * config.num_train_epochs
""" 模型准备 """
print("model name is {}".format(config.model_name))
if config.model_name == "BertOrigin":
from BertOrigin.BertOrigin import BertOrigin
model = BertOrigin.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels)
elif config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNN.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
elif config.model_name == "BertATT":
from BertATT.BertATT import BertATT
model = BertATT.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels)
elif config.model_name == "BertRCNN":
from BertRCNN.BertRCNN import BertRCNN
model = BertRCNN.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels)
elif config.model_name == "BertCNNPlus":
from BertCNNPlus.BertCNNPlus import BertCNNPlus
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNNPlus.from_pretrained(config.bert_model_dir,
cache_dir=config.cache_dir,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=gpu_ids)
""" 优化器准备 """
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=config.warmup_proportion,
t_total=num_train_optimization_steps)
""" 损失函数准备 """
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
train(config.num_train_epochs, n_gpu, model, train_dataloader,
dev_dataloader, optimizer, criterion,
config.gradient_accumulation_steps, device, label_list,
output_model_file, output_config_file, config.log_dir,
config.print_step, config.early_stop)
""" Test """
test_dataloader, _ = load_data(config.data_dir, tokenizer, processor,
config.max_seq_length,
config.test_batch_size, "test")
bert_config = BertConfig(output_config_file)
if config.model_name == "BertOrigin":
from BertOrigin.BertOrigin import BertOrigin
model = BertOrigin(bert_config, num_labels=num_labels)
elif config.model_name == "BertCNN":
from BertCNN.BertCNN import BertCNN
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNN(bert_config,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
elif config.model_name == "BertATT":
from BertATT.BertATT import BertATT
model = BertATT(bert_config, num_labels=num_labels)
elif config.model_name == "BertRCNN":
from BertRCNN.BertRCNN import BertRCNN
model = BertRCNN(bert_config, num_labels=num_labels)
elif config.model_name == "BertCNNPlus":
from BertCNNPlus.BertCNNPlus import BertCNNPlus
filter_sizes = [int(val) for val in config.filter_sizes.split()]
model = BertCNNPlus(bert_config,
num_labels=num_labels,
n_filters=config.filter_num,
filter_sizes=filter_sizes)
model.load_state_dict(torch.load(output_model_file))
model.to(device)
""" 损失函数准备 """
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
# test the model
eval_save(model, test_dataloader, criterion, device, label_list)
def main():
def evaluate(dataloader, export=None):
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_list = []
iter_idx = 0
corr_x = []
corr_y = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(
dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids,
segment_ids,
input_mask,
label_ids,
mse=is_float)
logits = model(input_ids,
segment_ids,
input_mask,
mse=is_float)
logits = logits.detach().cpu().numpy()
if export is not None:
logits_list.append(logits)
label_ids = label_ids.to('cpu').numpy()
if is_float:
corr_x.extend(logits.flatten())
corr_y.extend(label_ids.flatten())
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
# if (iter_idx + 1) % 1000 == 0 and export is not None:
# torch.save((iter_idx, logits_list), export)
iter_idx += 1
if export is not None:
torch.save(logits_list, export)
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
if is_float:
print(pearsonr(corr_x, corr_y))
print(spearmanr(corr_x, corr_y))
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
return result
local_rank = -1
parser = argparse.ArgumentParser()
parser.add_argument("--config", "-c", type=str, required=True)
args, _ = parser.parse_known_args()
options = argconf.options_from_json("confs/options.json")
config = argconf.config_from_json(args.config)
args = edict(argconf.parse_args(options, config))
print(f"Using config: {args}")
set_seed(args.seed)
args.do_train = args.do_train and not args.do_test_only
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
"sst2": SST2Processor,
'qnli': QnliProcessor,
'rte': RteProcessor,
"imdb": IMDBSentenceProcessor,
"raw_single":
IMDBSentenceProcessor, # This is not a mistake, just poor naming.
"qqp": QuoraProcessor,
"sts": STSProcessor,
"raw_sts_pair": RawSTSPairProcessor,
"raw_pair": RawPairProcessor
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
# if os.path.exists(args.workspace) and os.listdir(args.workspace) and args.do_train:
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.workspace))
if not os.path.exists(args.workspace):
os.makedirs(args.workspace)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = args.n_labels
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.model_file,
do_lower_case=args.uncased)
num_train_optimization_steps = None
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = os.path.join(PYTORCH_PRETRAINED_BERT_CACHE,
'distributed_{}'.format(local_rank))
model = BertForSequenceClassification.from_pretrained(
args.model_file, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
# sd = torch.load('qqp.pt')
# sd = torch.load('sts.pt')
# del sd['classifier.weight']
# del sd['classifier.bias']
# model.load_state_dict(sd, strict=False)
if local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
param_optimizer = list(
filter(
lambda x: x[0] in
("module.classifier.weight", "module.classifier.bias"),
param_optimizer))
print(len(param_optimizer))
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
train_features = convert_examples_to_features(train_examples, label_list,
args.max_seq_length,
tokenizer)
is_float = isinstance(train_features[0].label_id, float)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.float if is_float else torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
if local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
# BEGIN SST-2 -> QQP experiments
# END SST-2 -> QQP experiments
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids,
segment_ids,
input_mask,
label_ids,
mse=is_float)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
output_model_file = os.path.join(args.workspace, WEIGHTS_NAME)
if args.do_train:
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.workspace, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
elif args.do_test_only:
convert = convert_single_to_dp if isinstance(
model, torch.nn.DataParallel) else convert_dp_to_single
model.load_state_dict(convert(torch.load(output_model_file)))
else:
# pass
model = BertForSequenceClassification.from_pretrained(
args.model_file, num_labels=num_labels)
model.to(device)
if args.export:
model.eval()
train_dataloader = DataLoader(train_data,
batch_size=args.eval_batch_size,
shuffle=False)
with torch.no_grad():
evaluate(train_dataloader, export=args.export)
return
if args.visualize:
model.eval()
train_dataloader = DataLoader(train_data,
batch_size=args.eval_batch_size,
shuffle=False)
with open(os.path.join(args.workspace, "viz_results.csv"), "w") as f:
writer = None
if args.do_eval and (local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_test_examples(
args.data_dir
) if args.do_test_only else processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor(
[f.label_id for f in eval_features],
dtype=torch.long
if isinstance(eval_features[0].label_id, int) else torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
result = evaluate(eval_dataloader)
output_eval_file = os.path.join(args.workspace, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def __init__(self,
contextual_embedding_dim,
entity_embedding_dim: int,
entity_embeddings: torch.nn.Embedding,
max_sequence_length: int = 512,
span_encoder_config: Dict[str, int] = None,
dropout: float = 0.1,
output_feed_forward_hidden_dim: int = 100,
initializer_range: float = 0.02,
weighted_entity_threshold: float = None,
null_entity_id: int = None,
include_null_embedding_in_dot_attention: bool = False):
"""
Idea: Align the bert and KG vector space by learning a mapping between
them.
"""
super().__init__()
self.span_extractor = SelfAttentiveSpanExtractor(entity_embedding_dim)
init_bert_weights(self.span_extractor._global_attention._module,
initializer_range)
self.dropout = torch.nn.Dropout(dropout)
self.bert_to_kg_projector = torch.nn.Linear(contextual_embedding_dim,
entity_embedding_dim)
init_bert_weights(self.bert_to_kg_projector, initializer_range)
self.projected_span_layer_norm = BertLayerNorm(entity_embedding_dim,
eps=1e-5)
init_bert_weights(self.projected_span_layer_norm, initializer_range)
self.kg_layer_norm = BertLayerNorm(entity_embedding_dim, eps=1e-5)
init_bert_weights(self.kg_layer_norm, initializer_range)
# already pretrained, don't init
self.entity_embeddings = entity_embeddings
self.entity_embedding_dim = entity_embedding_dim
# layers for the dot product attention
if weighted_entity_threshold is not None or include_null_embedding_in_dot_attention:
if hasattr(self.entity_embeddings, 'get_null_embedding'):
null_embedding = self.entity_embeddings.get_null_embedding()
else:
null_embedding = self.entity_embeddings.weight[
null_entity_id, :]
else:
null_embedding = None
self.dot_attention_with_prior = DotAttentionWithPrior(
output_feed_forward_hidden_dim, weighted_entity_threshold,
null_embedding, initializer_range)
self.null_entity_id = null_entity_id
self.contextual_embedding_dim = contextual_embedding_dim
if span_encoder_config is None:
self.span_encoder = None
else:
# create BertConfig
assert len(span_encoder_config) == 4
config = BertConfig(
0, # vocab size, not used
hidden_size=span_encoder_config['hidden_size'],
num_hidden_layers=span_encoder_config['num_hidden_layers'],
num_attention_heads=span_encoder_config['num_attention_heads'],
intermediate_size=span_encoder_config['intermediate_size'])
self.span_encoder = BertEncoder(config)
init_bert_weights(self.span_encoder, initializer_range)
def main():
parser = setup_parser()
args = parser.parse_args()
# specifies the path where the biobert or clinical bert model is saved
if args.bert_model == 'biobert' or args.bert_model == 'clinical_bert':
args.bert_model = args.model_loc
print(args.bert_model)
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
"mednli": MedNLIProcessor
}
num_labels_task = {"cola": 2, "mnli": 3, "mrpc": 2, "mednli": 3}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
print('TRAIN')
train = processor.get_train_examples(args.data_dir)
print([(train[i].text_a, train[i].text_b, train[i].label)
for i in range(3)])
print('DEV')
dev = processor.get_dev_examples(args.data_dir)
print([(dev[i].text_a, dev[i].text_b, dev[i].label) for i in range(3)])
print('TEST')
test = processor.get_test_examples(args.data_dir)
print([(test[i].text_a, test[i].text_b, test[i].label) for i in range(3)])
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
PYTORCH_PRETRAINED_BERT_CACHE, 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
if args.do_train:
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_test and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running testing *****")
logger.info(" Num examples = %d", len(test_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=args.eval_batch_size)
model.eval()
test_loss, test_accuracy = 0, 0
nb_test_steps, nb_test_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
test_dataloader, desc="Testing"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_test_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_test_accuracy = accuracy(logits, label_ids)
test_loss += tmp_test_loss.mean().item()
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'test_loss': test_loss,
'test_accuracy': test_accuracy,
'global_step': global_step,
'loss': loss
}
output_test_file = os.path.join(args.output_dir, "test_results.txt")
with open(output_test_file, "w") as writer:
logger.info("***** Test results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
args = get_args()
args.train_datasets = args.train_datasets.split(',')
args.test_datasets = args.test_datasets.split(',')
args.output_dir = os.path.abspath(args.output_dir)
os.makedirs(args.output_dir, exist_ok=True)
# log_path = args.log_file
logger = create_logger(__name__, to_disk=True, log_file=args.log_file)
logger.info('args: %s', json.dumps(vars(args), indent=2))
set_environment(args.seed, args.cuda)
task_defs = BlueTaskDefs(args.task_def)
encoder_type = task_defs.encoder_type
assert encoder_type == EncoderModelType.BERT, '%s: only support BERT' % encoder_type
args.encoder_type = encoder_type
logger.info('Launching the MT-DNN training')
# update data dir
train_data_list = []
tasks = {}
tasks_class = {}
nclass_list = []
decoder_opts = []
task_types = []
dropout_list = []
for dataset in args.train_datasets:
task = dataset.split('_')[0]
if task in tasks:
logger.warning('Skipping: %s in %s', task, tasks)
continue
assert task in task_defs.n_class_map, \
'%s not in n_class_map: %s' % (task, task_defs.n_class_map)
assert task in task_defs.data_format_map, \
'%s not in data_format_map: %s' % (task, task_defs.data_format_map)
data_type = task_defs.data_format_map[task]
nclass = task_defs.n_class_map[task]
task_id = len(tasks)
if args.mtl_opt > 0:
task_id = tasks_class[nclass] if nclass in tasks_class else len(
tasks_class)
task_type = task_defs.task_type_map[task]
dopt = generate_decoder_opt(task_defs.enable_san_map[task],
args.answer_opt)
if task_id < len(decoder_opts):
decoder_opts[task_id] = min(decoder_opts[task_id], dopt)
else:
decoder_opts.append(dopt)
task_types.append(task_type)
if task not in tasks:
tasks[task] = len(tasks)
if args.mtl_opt < 1:
nclass_list.append(nclass)
if nclass not in tasks_class:
tasks_class[nclass] = len(tasks_class)
if args.mtl_opt > 0:
nclass_list.append(nclass)
dropout_p = task_defs.dropout_p_map.get(task, args.dropout_p)
dropout_list.append(dropout_p)
# use train and dev
train_path = os.path.join(args.data_dir, f'{dataset}_train+dev.json')
logger.info('Loading %s as task %s', task, task_id)
train_data = BatchGen(BatchGen.load(train_path,
True,
task_type=task_type,
maxlen=args.max_seq_len),
batch_size=args.batch_size,
dropout_w=args.dropout_w,
gpu=args.cuda,
task_id=task_id,
maxlen=args.max_seq_len,
data_type=data_type,
task_type=task_type,
encoder_type=encoder_type)
train_data_list.append(train_data)
dev_data_list = []
test_data_list = []
for dataset in args.test_datasets:
task = dataset.split('_')[0]
task_id = tasks_class[
task_defs.n_class_map[task]] if args.mtl_opt > 0 else tasks[task]
task_type = task_defs.task_type_map[task]
data_type = task_defs.data_format_map[task]
dev_path = os.path.join(args.data_dir, f'{dataset}_dev.json')
dev_data = BatchGen(BatchGen.load(dev_path,
False,
task_type=task_type,
maxlen=args.max_seq_len),
batch_size=args.batch_size_eval,
gpu=args.cuda,
is_train=False,
task_id=task_id,
maxlen=args.max_seq_len,
data_type=data_type,
task_type=task_type,
encoder_type=encoder_type)
dev_data_list.append(dev_data)
test_path = os.path.join(args.data_dir, f'{dataset}_test.json')
test_data = BatchGen(BatchGen.load(test_path,
False,
task_type=task_type,
maxlen=args.max_seq_len),
batch_size=args.batch_size_eval,
gpu=args.cuda,
is_train=False,
task_id=task_id,
maxlen=args.max_seq_len,
data_type=data_type,
task_type=task_type,
encoder_type=encoder_type)
test_data_list.append(test_data)
opt = copy.deepcopy(vars(args))
opt['answer_opt'] = decoder_opts
opt['task_types'] = task_types
opt['tasks_dropout_p'] = dropout_list
label_size = ','.join([str(l) for l in nclass_list])
opt['label_size'] = label_size
logger.info('#' * 20)
logger.info('opt: %s', json.dumps(opt, indent=2))
logger.info('#' * 20)
bert_model_path = args.init_checkpoint
state_dict = None
if os.path.exists(bert_model_path):
state_dict = torch.load(bert_model_path)
config = state_dict['config']
config['attention_probs_dropout_prob'] = args.bert_dropout_p
config['hidden_dropout_prob'] = args.bert_dropout_p
opt.update(config)
else:
logger.error('#' * 20)
logger.error('Could not find the init model!\n'
'The parameters will be initialized randomly!')
logger.error('#' * 20)
config = BertConfig(vocab_size_or_config_json_file=30522).to_dict()
opt.update(config)
all_iters = [iter(item) for item in train_data_list]
all_lens = [len(bg) for bg in train_data_list]
# div number of grad accumulation.
num_all_batches = args.epochs * sum(
all_lens) // args.grad_accumulation_step
logger.info('############# Gradient Accumulation Info #############')
logger.info('number of step: %s', args.epochs * sum(all_lens))
logger.info('number of grad grad_accumulation step: %s',
args.grad_accumulation_step)
logger.info('adjusted number of step: %s', num_all_batches)
logger.info('############# Gradient Accumulation Info #############')
if len(train_data_list) > 1 and args.ratio > 0:
num_all_batches = int(args.epochs * (len(train_data_list[0]) *
(1 + args.ratio)))
model = MTDNNModel(opt,
state_dict=state_dict,
num_train_step=num_all_batches)
if args.resume and args.model_ckpt:
logger.info('loading model from %s', args.model_ckpt)
model.load(args.model_ckpt)
# model meta str
headline = '############# Model Arch of MT-DNN #############'
# print network
logger.debug('\n{}\n{}\n'.format(headline, model.network))
# dump config
config_file = os.path.join(args.output_dir, 'config.json')
with open(config_file, 'a', encoding='utf-8') as writer:
writer.write('{}\n'.format(json.dumps(opt)))
writer.write('\n{}\n{}\n'.format(headline, model.network))
logger.info("Total number of params: %s", model.total_param)
# tensorboard
if args.tensorboard:
args.tensorboard_logdir = os.path.join(args.output_dir,
args.tensorboard_logdir)
tensorboard = SummaryWriter(log_dir=args.tensorboard_logdir)
for epoch in range(0, args.epochs):
logger.warning('At epoch %s', epoch)
for train_data in train_data_list:
train_data.reset()
start = datetime.now()
all_indices = []
if len(train_data_list) > 1 and args.ratio > 0:
main_indices = [0] * len(train_data_list[0])
extra_indices = []
for i in range(1, len(train_data_list)):
extra_indices += [i] * len(train_data_list[i])
random_picks = int(
min(len(train_data_list[0]) * args.ratio, len(extra_indices)))
extra_indices = np.random.choice(extra_indices,
random_picks,
replace=False)
if args.mix_opt > 0:
extra_indices = extra_indices.tolist()
random.shuffle(extra_indices)
all_indices = extra_indices + main_indices
else:
all_indices = main_indices + extra_indices.tolist()
else:
for i in range(1, len(train_data_list)):
all_indices += [i] * len(train_data_list[i])
if args.mix_opt > 0:
random.shuffle(all_indices)
all_indices += [0] * len(train_data_list[0])
if args.mix_opt < 1:
random.shuffle(all_indices)
for i in range(len(all_indices)):
task_id = all_indices[i]
batch_meta, batch_data = next(all_iters[task_id])
model.update(batch_meta, batch_data)
if model.local_updates % (args.log_per_updates * args.grad_accumulation_step) == 0 \
or model.local_updates == 1:
remaining_time = str((datetime.now() - start) / (i + 1) *
(len(all_indices) - i - 1)).split('.')[0]
logger.info(
'Task [%2d] updates[%6d] train loss[%.5f] remaining[%s]',
task_id, model.updates, model.train_loss.avg,
remaining_time)
if args.tensorboard:
tensorboard.add_scalar('train/loss',
model.train_loss.avg,
global_step=model.updates)
if args.save_per_updates_on \
and (model.local_updates % (
args.save_per_updates * args.grad_accumulation_step) == 0):
model_file = os.path.join(args.output_dir,
f'model_{epoch}_{model.updates}.pt')
logger.info('Saving mt-dnn model to %s', model_file)
model.save(model_file)
for idx, dataset in enumerate(args.test_datasets):
task = dataset.split('_')[0]
label_mapper = task_defs.label_mapper_map[task]
metric_meta = task_defs.metric_meta_map[task]
# dev
data = dev_data_list[idx]
with torch.no_grad():
metrics, predictions, scores, golds, ids = eval_model(
model, data, metric_meta, args.cuda, True, label_mapper)
for key, val in metrics.items():
if args.tensorboard:
tensorboard.add_scalar(f'dev/{dataset}/{key}',
val,
global_step=epoch)
logger.warning('Task %s - epoch %s - Dev %s: %s', dataset,
epoch, key, val)
path = os.path.join(args.output_dir,
f'{dataset}_dev_scores_{epoch}.json')
result = {
'metrics': metrics,
'predictions': predictions,
'uids': ids,
'scores': scores
}
dump(path, result)
path = os.path.join(args.output_dir,
f'{dataset}_dev_scores_{epoch}_2.json')
dump2(path, ids, scores, predictions)
# test
data = test_data_list[idx]
with torch.no_grad():
metrics, predictions, scores, golds, ids = eval_model(
model, data, metric_meta, args.cuda, True, label_mapper)
for key, val in metrics.items():
if args.tensorboard:
tensorboard.add_scalar(f'test/{dataset}/{key}',
val,
global_step=epoch)
logger.warning('Task %s - epoch %s - Test %s: %s', dataset,
epoch, key, val)
path = os.path.join(args.output_dir,
f'{dataset}_test_scores_{epoch}.json')
result = {
'metrics': metrics,
'predictions': predictions,
'uids': ids,
'scores': scores
}
dump(path, result)
path = os.path.join(args.output_dir,
f'{dataset}_test_scores_{epoch}_2.json')
dump2(path, ids, scores, predictions)
logger.info('[new test scores saved.]')
if not args.not_save:
model_file = os.path.join(args.output_dir, f'model_{epoch}.pt')
model.save(model_file)
if args.tensorboard:
tensorboard.close()
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=512,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=5,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=5,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=10.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
"recipes": RecipesProcessor,
}
num_labels_task = {
"cola": 2,
"mnli": 3,
"mrpc": 2,
"recipes": 2,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
PYTORCH_PRETRAINED_BERT_CACHE, 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification2.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_mean_pooling_ids = torch.tensor(
[f.mean_pooling_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_mean_pooling_ids,
all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for ep in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, mean_pooling_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask,
mean_pooling_ids, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation after epoch %d *****", ep)
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor(
[f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor(
[f.segment_ids for f in eval_features], dtype=torch.long)
all_mean_pooling_ids = torch.tensor(
[f.mean_pooling_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_mean_pooling_ids,
all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, mean_pooling_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
mean_pooling_ids = mean_pooling_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
mean_pooling_ids, label_ids)
logits = model(input_ids, segment_ids, input_mask,
mean_pooling_ids)
logits = logits.detach().cpu().numpy()
np.save('./bert_whole_ep{}.npy'.format(ep), logits)
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir,
"eval_results_ep{}.txt".format(ep))
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_train:
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_mean_pooling_ids = torch.tensor(
[f.mean_pooling_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_mean_pooling_ids,
all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, mean_pooling_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
mean_pooling_ids = mean_pooling_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
mean_pooling_ids, label_ids)
logits = model(input_ids, segment_ids, input_mask,
mean_pooling_ids)
logits = logits.detach().cpu().numpy()
np.save('./bert_whole.npy', logits)
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main(config, bert_vocab_file, bert_model_dir):
if not os.path.exists(config.output_dir):
os.makedirs(config.output_dir)
if not os.path.exists(config.cache_dir):
os.makedirs(config.cache_dir)
output_model_file = os.path.join(config.output_dir,
config.weights_name) # 模型输出文件
output_config_file = os.path.join(config.output_dir, config.config_name)
# 设备准备
gpu_ids = [int(device_id) for device_id in config.gpu_ids.split()]
device, n_gpu = get_device(gpu_ids[0])
if n_gpu > 1:
n_gpu = len(gpu_ids)
config.train_batch_size = config.train_batch_size // config.gradient_accumulation_steps
""" 设定随机种子 """
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(config.seed)
tokenizer = BertTokenizer.from_pretrained(
bert_vocab_file, do_lower_case=config.do_lower_case)
label_list = ["0", "1", "2", "3"]
if config.do_train:
# 数据准备
train_file = os.path.join(config.data_dir, "train.json")
dev_file = os.path.join(config.data_dir, "dev.json")
train_dataloader, train_len = load_data(train_file, tokenizer,
config.max_seq_length,
config.train_batch_size)
dev_dataloader, dev_len = load_data(dev_file, tokenizer,
config.max_seq_length,
config.dev_batch_size)
num_train_steps = int(train_len / config.train_batch_size /
config.gradient_accumulation_steps *
config.num_train_epochs)
# 模型准备
model = BertForMultipleChoice.from_pretrained(
bert_model_dir, cache_dir=config.cache_dir, num_choices=4)
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=gpu_ids)
# 优化器准备
param_optimizer = list(model.named_parameters())
param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=config.warmup_proportion,
t_total=num_train_steps)
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
train(config.num_train_epochs, n_gpu, train_dataloader, dev_dataloader,
model, optimizer, criterion, config.gradient_accumulation_steps,
device, label_list, output_model_file, output_config_file,
config.log_dir, config.print_step)
test_file = os.path.join(config.data_dir, "test.json")
test_dataloader, _ = load_data(test_file, tokenizer, config.max_seq_length,
config.test_batch_size)
bert_config = BertConfig(output_config_file)
model = BertForMultipleChoice(bert_config, num_choices=len(label_list))
model.load_state_dict(torch.load(output_model_file))
model.to(device)
""" 损失函数准备 """
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
test_loss, test_acc, test_report = evaluate(model, test_dataloader,
criterion, device, label_list)
print("-------------- Test -------------")
print(f'\t Loss: {test_loss: .3f} | Acc: {test_acc*100: .3f} %')
for label in label_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
print_list = ['macro avg', 'weighted avg']
for label in print_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
