def __init__(self):
self.bert_config = BertConfig.from_json_file(
os.path.join(path, 'uncased_L-12_H-768_A-12/bert_config.json'))
self.max_sequence_length = 128
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
if self.max_sequence_length > self.bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(self.max_sequence_length,
self.bert_config.max_position_embeddings))
self.processor = LogicProcessor()
self.label_list = self.processor.get_labels()
self.tokenizer = tokenization.FullTokenizer(vocab_file=os.path.join(
path, 'uncased_L-12_H-768_A-12/vocab.txt'),
do_lower_case=False)
self.model = BertForSequenceClassification(self.bert_config,
len(self.label_list))
init_checkpoint = os.path.join(path, 'model/logic_model_500.bin')
#Future save model Load code
if init_checkpoint is not None:
self.model.load_state_dict(
torch.load(init_checkpoint, map_location='cpu'))
self.model.to(self.device)
def predict_single_sentence(model_wieght_path, text):
# initialize basic options
random.seed(42)
np.random.seed(42)
torch.manual_seed(42)
output_mode = 'classification'
processor = PuriProcessor()
label_list = processor.get_labels()
num_labels = len(label_list)
bert_model = 'bert-base-multilingual-uncased'
tokenizer = BertTokenizer.from_pretrained(bert_model, do_lower_case=True)
# convert text to feature
example = processor.create_single_example(text)
feature = convert_single_example_to_feature(example, 128, tokenizer,
output_mode)
# create_model
model = BertForSequenceClassification.from_pretrained(
model_wieght_path, num_labels=num_labels)
model.eval()
preds = []
input_ids = torch.tensor(feature.input_ids, dtype=torch.long).unsqueeze(0)
input_mask = torch.tensor(feature.input_mask,
dtype=torch.long).unsqueeze(0)
segment_ids = torch.tensor(feature.segment_ids,
dtype=torch.long).unsqueeze(0)
# label_ids = feature.label_ids
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask, labels=None)
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0], logits.detach().cpu().numpy(), axis=0)
preds = preds[0]
result = np.argmax(preds, axis=1)
return preds, result
def infer(input, target_start_id, target_end_id, args):
sent = input.split(" ")
assert 0 <= target_start_id and target_start_id < target_end_id and target_end_id <= len(
sent)
target = " ".join(sent[target_start_id:target_end_id])
device = torch.device(
"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
label_list = ["0", "1"]
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=True)
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
print(f"input: {input}\ntarget: {target}")
examples = construct_context_gloss_pairs_through_nltk(
input, target_start_id, target_end_id)
eval_features, candidate_results = convert_to_features(examples, tokenizer)
input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
model.eval()
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
with torch.no_grad():
logits = model(input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
labels=None)
logits_ = F.softmax(logits, dim=-1)
logits_ = logits_.detach().cpu().numpy()
output = np.argmax(logits_, axis=0)[1]
print(f"results:\ngloss: {candidate_results[output][1]}")
def main():
# args = parse_arguments()
# del args.local_rank
# print(args)
# args_to_yaml(args, 'config_finetune_train_glue_mrpc.yaml')
# exit(0)
config_yaml, local_rank = parse_my_arguments()
args = args_from_yaml(config_yaml)
args.local_rank = local_rank
""" Experiment Setup """
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
print(
"WARNING: Output directory ({}) already exists and is not empty.".
format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
}
num_labels_task = {
"cola": 2,
"mnli": 3,
"mrpc": 2,
}
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % task_name)
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
""" Prepare Model """
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
PYTORCH_PRETRAINED_BERT_CACHE, 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
state_dict = state_dict.get(
'model', state_dict
) # in a full checkpoint weights are saved in state_dict['model']
model.load_state_dict(state_dict, strict=False)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
plain_model = getattr(model, 'module', model)
with open(args.sparsity_config, 'r') as f:
raw_dict = yaml.load(f, Loader=yaml.SafeLoader)
masks = dict.fromkeys(raw_dict['prune_ratios'].keys())
for param_name in list(masks.keys()):
if get_parameter_by_name(plain_model, param_name) is None:
print(f'[WARNING] Cannot find {param_name}')
del masks[param_name]
for param_name in masks:
param = get_parameter_by_name(plain_model, param_name)
non_zero_mask = torch.ne(param, 0).to(param.dtype)
masks[param_name] = non_zero_mask
""" Prepare Optimizer"""
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.fp16_utils.fp16_optimizer import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
""" Prepare Dataset """
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
""" Training Loop """
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
if args.max_steps > 0 and global_step > args.max_steps:
break
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
plain_model = getattr(model, 'module', model)
for param_name, mask in masks.items():
get_parameter_by_name(plain_model,
param_name).data *= mask
""" Load Model for Evaluation """
if args.do_train:
# Save a trained model and the associated configuration
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
if is_main_process(
): # only the main process should save the trained model
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
torch.save(model_to_save.state_dict(), output_model_file)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
state_dict = state_dict.get('model', state_dict)
model.load_state_dict(state_dict, strict=False)
model.to(device)
""" Run Evaluation """
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
from optimization import BERTAdam
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score, matthews_corrcoef
import matplotlib.pyplot as plt
import tkinter as tk
df = pd.read_csv('/Users/iluzaangirova/Ilyuza/dataset_5.csv', encoding='utf8')
df = df[['topics', 'title']]
maps = pd.factorize(df.topics)[1]
device = 'cpu'
bert_config = BertConfig.from_json_file(
'/Users/iluzaangirova/Ilyuza/bert_config.json')
model = BertForSequenceClassification(bert_config, 5)
model.to(device)
model = torch.nn.DataParallel(model)
model.load_state_dict(
torch.load('/Users/iluzaangirova/Ilyuza/weights5000.pth',
map_location='cpu'))
tokenizer = tokenization.FullTokenizer(
vocab_file='/Users/iluzaangirova/Ilyuza/vocab.txt', do_lower_case=False)
def predict(sentence, tokenizer, max_len):
# print(sentence)
tokens_a = tokenizer.tokenize(sentence)
if len(tokens_a) > max_len - 2:
tokens_a = tokens_a[0:(max_len - 2)]
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .csv files (or other data files) for the task."
)
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
required=True,
help="Initial checkpoint (usually from a pre-trained BERT model).")
## Other parameters
parser.add_argument("--eval_test",
default=False,
action='store_true',
help="Whether to run eval on the test set.")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10% of training.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument('--save_model',
type=str,
default=None,
help="Path to save the trained model to")
parser.add_argument('--load_model',
type=str,
default=None,
help="Path to load the trained model from")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
# prepare dataloaders
processor = processor_class()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
# training set
train_examples = None
num_train_steps = None
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs)
train_features = convert_examples_to_features(train_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
# test set
if args.eval_test:
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
test_dataloader = DataLoader(test_data,
batch_size=args.eval_batch_size,
shuffle=False)
del all_input_ids, all_input_mask, all_label_ids, all_segment_ids
# model and optimizer
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.01
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
# train
output_log_file = os.path.join(args.output_dir, "log.txt")
print("output_log_file=", output_log_file)
with open(output_log_file, "w") as writer:
if args.eval_test:
writer.write(
"epoch\tglobal_step\tloss\ttest_loss\ttest_accuracy\n")
else:
writer.write("epoch\tglobal_step\tloss\n")
if args.load_model:
checkpoint = torch.load(args.load_model)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
global_step = checkpoint['global_step']
print("===Loaded previous checkpoint===")
for key in checkpoint:
print(key, "=", checkpoint[key])
else:
global_step = 0
epoch = 0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
epoch += 1
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
# eval_test
if args.eval_test:
model.eval()
test_loss, test_accuracy = 0, 0
nb_test_steps, nb_test_examples = 0, 0
with open(
os.path.join(args.output_dir,
"test_ep_" + str(epoch) + ".txt"),
"w") as f_test:
for input_ids, input_mask, segment_ids, label_ids in test_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_test_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids)
logits = F.softmax(logits, dim=-1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
outputs = np.argmax(logits, axis=1)
for output_i in range(len(outputs)):
f_test.write(str(outputs[output_i]))
for ou in logits[output_i]:
f_test.write(" " + str(ou))
f_test.write("\n")
tmp_test_accuracy = np.sum(outputs == label_ids)
test_loss += tmp_test_loss.mean().item()
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
result = collections.OrderedDict()
if args.eval_test:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps,
'test_loss': test_loss,
'test_accuracy': test_accuracy
}
else:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
logger.info("***** Eval results *****")
with open(output_log_file, "a+") as writer:
for key in result.keys():
logger.info(" %s = %s\n", key, str(result[key]))
writer.write("%s\t" % (str(result[key])))
writer.write("\n")
if args.save_model:
try:
torch.save(
{
'model_state_dict': model.state_dict(),
'epoch': epoch,
'optimizer_state_dict': optimizer.state_dict(),
'loss': tr_loss / nb_tr_steps,
'global_step': global_step,
},
f=args.save_model)
except FileNotFoundError:
f = open(args.save_model, "w+")
torch.save(
{
'model_state_dict': model.state_dict(),
'epoch': epoch,
'optimizer_state_dict': optimizer.state_dict(),
'loss': tr_loss / nb_tr_steps,
'global_step': global_step,
},
f=f)
def __init__(self, args):
self.opt = args
self.writer = SummaryWriter(log_dir=self.opt.output_dir) # tensorboard
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length,
bert_config.max_position_embeddings))
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
# os.makedirs(args.output_dir, exist_ok=True)
self.dataset = ReadData(self.opt) # Read the data and preprocess it
self.num_train_steps = None
self.num_train_steps = int(
len(self.dataset.train_examples) / self.opt.train_batch_size /
self.opt.gradient_accumulation_steps * self.opt.num_train_epochs)
self.opt.label_size = len(self.dataset.label_list)
args.output_dim = len(self.dataset.label_list)
print("label size: {}".format(args.output_dim))
# 初始化模型
print("initialize model ...")
if args.model_class == BertForSequenceClassification:
self.model = BertForSequenceClassification(
bert_config, len(self.dataset.label_list))
else:
self.model = model_classes[args.model_name](bert_config, args)
if args.init_checkpoint is not None:
self.model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
self.model.half()
# 冻结参数
# for name, p in self.model.named_parameters():
# if name.startswith('bert.encoder.layer.11') or name.startswith('bert.encoder.layer.10') or name.startswith('bert.encoder.layer.9') or name.startswith('bert.encoder.layer.8'): # 冻结最后一层
# p.requires_grad = False
# 计算模型的参数个数
n_trainable_params, n_nontrainable_params = 0, 0
for p in self.model.parameters():
n_params = torch.prod(torch.tensor(
p.shape)) # torch.prod()表示计算所有元素的乘积
if p.requires_grad: # 是否需要求梯度
n_trainable_params += n_params
else:
n_nontrainable_params += n_params
print('n_trainable_params: {0}, n_nontrainable_params: {1}'.format(
n_trainable_params, n_nontrainable_params))
self.model.to(args.device)
# 并行化
if self.opt.n_gpu > 1:
self.model = torch.nn.DataParallel(self.model,
device_ids=self.opt.gpu_id)
# Prepare optimizer
if args.fp16:
self.param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in self.model.named_parameters()]
elif args.optimize_on_cpu:
self.param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in self.model.named_parameters()]
else:
self.param_optimizer = list(self.model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [{
'params':
[p for n, p in self.param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in self.param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
self.optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=self.num_train_steps)
# 配置自己模型的优化器
# [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')]
# self.optimizer_me = torch.optim.Adam(
# [{'params': [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')]}], lr=0.001,
# weight_decay=0)
# self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer_me, mode='max',
# patience=3) # 3个epoch后，所监测的值停止增加时自动调整学习率
self.global_step = 0 # 初始化全局步数为 0
self.max_test_acc = 0
self.max_test_f1 = 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--train_file",
default=None,
type=str,
required=True,
help="The train file path")
parser.add_argument("--eval_file",
default=None,
type=str,
required=True,
help="The dev file path")
parser.add_argument("--predict_file",
default=None,
type=str,
required=False,
help="The predict file path")
parser.add_argument("--predict_result_file",
default='datas/result.csv',
type=str,
required=False,
help="The predict result file path")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=250,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--load_checkpoint",
default=False,
action='store_true',
help="Whether to run load checkpoint.")
parser.add_argument("--num_labels",
default=1,
type=int,
help="mapping classify nums")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--epoches",
default=6,
type=int,
help="Total epoch numbers for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=6.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
vocab_path = os.path.join(args.bert_model, VOCAB_NAME)
# bert_config = BertConfig.from_json_file(vocab_path)
data_processor = DataProcessor()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# if args.do_train:
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(
# args.output_dir))
# else:
# os.makedirs(args.output_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path,
do_lower_case=args.do_lower_case)
model = BertForSequenceClassification.from_pretrained(args.bert_model,
num_labels=3)
for k, v in model.state_dict().items():
print(f'k = {k}, v.grad = {v.grad}')
model.to(device)
# model = torch.nn.DataParallel(model)
def evaluating(model, eval_dataloader):
model.eval()
eval_loss = 0
logits, labels = [], []
for step, batch in enumerate(eval_dataloader):
input_ids, input_mask, segment_ids, label_ids = [
b.to(device) for b in batch
]
with torch.no_grad():
loss, logit = model(input_ids, segment_ids, input_mask,
label_ids)
loss = loss.mean()
eval_loss = loss * args.gradient_accumulation_steps if step == 0 else eval_loss + loss * args.gradient_accumulation_steps
logit = torch.argmax(logit, dim=-1)
logits.extend(logit.tolist())
labels.extend(label_ids.tolist())
return (eval_loss.item() / step, logits, labels)
def predicting(model, dataloader):
model.eval()
logits, example_ids = [], []
for step, batch in enumerate(dataloader):
if step % 100 == 0:
print(f'当前预测进度： {step}/{len(dataloader)}')
input_ids, input_mask, segment_ids, label_ids = [
b.to(device) for b in batch
]
with torch.no_grad():
logit = model(input_ids, segment_ids, input_mask)
logit = torch.argmax(logit, dim=-1)
logits.extend(logit.tolist())
example_ids.extend(label_ids.tolist())
return logits, example_ids
def eval_meric(model, data_loader):
eval_loss, all_logits, all_labels = evaluating(model, data_loader)
accuracy(all_labels, all_logits)
logger.info(f'Average eval loss = {eval_loss}')
return eval_loss
def write_predict_file(model, data_loader, file_path):
"""
写入预测文件： 格式：'五彩滨云-final.csv'
"""
logits, ids = predicting(model, data_loader)
assert len(ids) == len(logits)
logger.info(
f'zero nums {logits.count(0)}, one nums {logits.count(1)}, two nums {logits.count(2)}'
)
labels = [
data_processor.eval_dict[id][1] for id, logit in zip(ids, logits)
]
# if not args.do_eval:
# logits = [i - 1 for i in logits]
# data_df = pd.DataFrame({'id': ids, 'y': logits})
# else:
assert len(labels) == len(logits)
# accuracy(labels, logits)
passages = [
data_processor.eval_dict[id][0] for id, logit in zip(ids, logits)
]
autors = [
data_processor.eval_dict[id][2] for id, logit in zip(ids, logits)
]
like_counts = [
data_processor.eval_dict[id][3] for id, logit in zip(ids, logits)
]
times = [
data_processor.eval_dict[id][4] for id, logit in zip(ids, logits)
]
assert len(labels) == len(passages)
match_array = np.array((logits)) == np.array(labels)
match_list = match_array.tolist()
data_df = pd.DataFrame({
'id': ids,
'pred': logits,
'time': times,
'match': '',
'autors': autors,
'like_counts': like_counts,
'passage': passages
})
data_df.to_csv(file_path, index=None)
eval_examples = data_processor.get_examples(args.eval_file,
data_type='eval')
eval_features = convert_examples_to_features(args, eval_examples,
args.max_seq_length,
tokenizer)
eval_loader = ParaDataloader(eval_features)
eval_loader = DataLoader(eval_loader,
shuffle=False,
batch_size=args.eval_batch_size)
if 0:
# 数据读取
train_examples = data_processor.get_examples(args.train_file,
data_type='train')
# 特征转换
train_features = convert_examples_to_features(args, train_examples,
args.max_seq_length,
tokenizer)
num_train_steps = int(
len(train_features) // args.train_batch_size //
args.gradient_accumulation_steps * args.num_train_epochs)
# 数据loader
train_loader = ParaDataloader(train_features)
# 数据并行loader输入格式
train_loader = DataLoader(train_loader,
shuffle=True,
batch_size=args.train_batch_size)
model.zero_grad()
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
param_optimizer = list(model.named_parameters())
optimizer_grouped_parameters = [{
'params': [p for n, p in param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
tr_loss = None
for epoch in range(args.epoches):
model.train()
min_eval_loss = 10000
for step, batch in enumerate(train_loader):
input_ids, input_mask, segment_ids, label_ids = [
b.to(device) for b in batch
]
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
loss = loss.mean()
print(f'loss = {loss}')
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss = loss * args.gradient_accumulation_steps if step == 0 else tr_loss + loss * args.gradient_accumulation_steps
optimizer.step()
optimizer.zero_grad()
if step % 1000 == 1:
eval_loss = eval_meric(model, eval_loader)
if eval_loss < min_eval_loss:
save_checkpoint(model, epoch, args.output_dir)
if args.do_predict:
if args.load_checkpoint:
state_dict = torch.load('output/pytorch_model-0004.bin')
model.load_state_dict(state_dict)
logger.info(f'Start to predict......')
if args.do_eval:
predict_examples = data_processor.get_eval_examples(args.eval_file)
else:
predict_examples = data_processor.get_predict_examples(
args.predict_file)
predict_features = convert_examples_to_features(
args, predict_examples, args.max_seq_length, tokenizer)
predict_loader = ParaDataloader(predict_features)
predict_loader = DataLoader(predict_loader,
shuffle=False,
batch_size=args.eval_batch_size)
write_predict_file(model, predict_loader, args.predict_result_file)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_model", default=None, type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.")
## Other parameters
parser.add_argument("--cache_dir",
default="",
type=str,
help="Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument("--max_seq_length",
default=128,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion",
default=0.1,
type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument('--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale',
type=float, default=0,
help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
args.device = device
logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
model = BertForSequenceClassification.from_pretrained(args.bert_model, num_labels=num_labels)
if args.local_rank == 0:
torch.distributed.barrier()
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model,
device_ids=[args.local_rank],
output_device=args.local_rank,
find_unused_parameters=True)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
if args.local_rank in [-1, 0]:
tb_writer = SummaryWriter()
# Prepare data loader
train_examples = processor.get_train_examples(args.data_dir)
cached_train_features_file = os.path.join(args.data_dir, 'train_{0}_{1}_{2}'.format(
list(filter(None, args.bert_model.split('/'))).pop(),
str(args.max_seq_length),
str(task_name)))
try:
with open(cached_train_features_file, "rb") as reader:
train_features = pickle.load(reader)
except:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer, output_mode)
if args.local_rank == -1 or torch.distributed.get_rank() == 0:
logger.info(" Saving train features into cached file %s", cached_train_features_file)
with open(cached_train_features_file, "wb") as writer:
pickle.dump(train_features, writer)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)
warmup_linear = WarmupLinearSchedule(warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
# define a new function to compute loss values for both output_modes
logits = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step, args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
if args.local_rank in [-1, 0]:
tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
tb_writer.add_scalar('loss', loss.item(), global_step)
### Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
### Example:
if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
# Save a trained model, configuration and tokenizer
model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
# Load a trained model and vocabulary that you have fine-tuned
model = BertForSequenceClassification.from_pretrained(args.output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
# Good practice: save your training arguments together with the trained model
output_args_file = os.path.join(args.output_dir, 'training_args.bin')
torch.save(args, output_args_file)
else:
model = BertForSequenceClassification.from_pretrained(args.bert_model, num_labels=num_labels)
model.to(device)
### Evaluation
if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
cached_eval_features_file = os.path.join(args.data_dir, 'dev_{0}_{1}_{2}'.format(
list(filter(None, args.bert_model.split('/'))).pop(),
str(args.max_seq_length),
str(task_name)))
try:
with open(cached_eval_features_file, "rb") as reader:
eval_features = pickle.load(reader)
except:
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
if args.local_rank == -1 or torch.distributed.get_rank() == 0:
logger.info(" Saving eval features into cached file %s", cached_eval_features_file)
with open(cached_eval_features_file, "wb") as writer:
pickle.dump(eval_features, writer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# Run prediction for full data
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data) # Note that this sampler samples randomly
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
out_label_ids = None
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
out_label_ids = label_ids.detach().cpu().numpy()
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(
out_label_ids, label_ids.detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
elif output_mode == "regression":
preds = np.squeeze(preds)
result = compute_metrics(task_name, preds, out_label_ids)
loss = tr_loss/global_step if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
# hack for MNLI-MM
if task_name == "mnli":
task_name = "mnli-mm"
processor = processors[task_name]()
if os.path.exists(args.output_dir + '-MM') and os.listdir(args.output_dir + '-MM') and args.do_train:
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir + '-MM'):
os.makedirs(args.output_dir + '-MM')
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
out_label_ids = None
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=None)
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
out_label_ids = label_ids.detach().cpu().numpy()
else:
preds[0] = np.append(
preds[0], logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(
out_label_ids, label_ids.detach().cpu().numpy(), axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
preds = np.argmax(preds, axis=1)
result = compute_metrics(task_name, preds, out_label_ids)
loss = tr_loss/global_step if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir + '-MM', "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--input_text',
type=str,
default='',
help="Input text.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
args.device = device
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
if args.local_rank == 0:
torch.distributed.barrier()
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank,
find_unused_parameters=True)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
# Load a trained model and vocabulary that you have fine-tuned
model = BertForSequenceClassification.from_pretrained(
args.output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
# model = BertForSequenceClassification.from_pretrained(args.bert_model, num_labels=num_labels)
model.to(device)
### Evaluation
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
print("input text: ", args.input_text)
eval_examples = processor.get_test_example(args.input_text)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.float)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(
eval_data) # Note that this sampler samples randomly
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
out_label_ids = None
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
out_label_ids = label_ids.detach().cpu().numpy()
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
out_label_ids = np.append(out_label_ids,
label_ids.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
print(preds.shape)
print("preds", preds)
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
elif output_mode == "regression":
preds = np.squeeze(preds)
def main():
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
writer = SummaryWriter(os.path.join(args.output_dir, 'events'))
cache_dir = args.cache_dir
device = torch.device("cuda")
n_gpu = torch.cuda.device_count()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
save_code_log_path = args.output_dir
mkdir(args.output_dir)
logging.basicConfig(format='%(message)s',
datefmt='%m/%d/%Y %H:%M',
level=logging.INFO,
handlers=[
logging.FileHandler("{0}/{1}.log".format(
save_code_log_path, 'output')),
logging.StreamHandler()
])
logger.info(args)
logger.info("Command is: %s" % ' '.join(sys.argv))
logger.info("Device: {}, n_GPU: {}".format(device, n_gpu))
logger.info(
"Datasets are loaded from {}\nOutputs will be saved to {}\n".format(
args.data_dir, args.output_dir))
processor = LpaProcessor()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
load_dir = args.load_dir if args.load_dir else args.bert_model
logger.info('Model is loaded from %s' % load_dir)
model = BertForSequenceClassification.from_pretrained(
load_dir, cache_dir=cache_dir, num_labels=args.num_labels)
model.to(device)
if n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.do_train:
logger.info("\n************ Start Training *************")
run_train(device, processor, tokenizer, model, writer, phase="train")
if args.do_eval:
run_eval(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="dev")
run_eval(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="std_test")
if args.do_complex_test:
run_eval(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="complex_test")
if args.do_small_test:
run_eval(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="small_test")
if args.do_simple_test:
run_eval(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="simple_test")
if args.do_gen:
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="small_test")
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="train")
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="dev")
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="std_test")
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="simple_test")
evalSelectMostProb(device,
processor,
tokenizer,
model,
writer,
global_step=0,
save_detailed_res=True,
tensorboard=False,
phase="complex_test")
def convert():
# Initialise PyTorch model
config = BertConfig.from_json_file(args.bert_config_file)
# Load weights from TF model
path = args.tf_checkpoint_path
print("Converting TensorFlow checkpoint from {}".format(path))
init_vars = tf.train.list_variables(path)
names = []
arrays = []
classifier_classes = 0
for name, shape in init_vars:
print("Loading {} with shape {}".format(name, shape))
array = tf.train.load_variable(path, name)
print("Numpy array shape {}".format(array.shape))
names.append(name)
if name == 'output_bias':
classifier_classes = int(array.shape[0])
arrays.append(array)
if args.bert_model_type == 'classifier':
model = BertForSequenceClassification(config, classifier_classes)
else:
model = BertModel(config)
for name, array in zip(names, arrays):
first_name = name.split('/')[0]
print("Parsing {}".format(name))
name = name[5:] # skip "bert/"
name = name.split('/')
if name[0] in ['redictions', 'eq_relationship']:
print("Skipping", name[0])
continue
if 'adam' in name[-1]:
print("Skipping adam")
continue
if first_name == 'global_step':
# global step
print("Skipping global_step")
continue
if first_name in ['output_weights', 'output_bias']:
# final classifier layer output
name = [first_name]
pointer = model
if args.bert_model_type == 'classifier':
pointer = pointer.bert
for m_name in name:
if m_name == 'output_weights':
pointer = model.classifier.weight
break
if m_name == 'output_bias':
pointer = model.classifier.bias
break
if re.fullmatch(r'[A-Za-z]+_\d+', m_name):
l = re.split(r'_(\d+)', m_name)
else:
l = [m_name]
if l[0] == 'kernel':
pointer = getattr(pointer, 'weight')
else:
pointer = getattr(pointer, l[0])
if len(l) >= 2:
num = int(l[1])
pointer = pointer[num]
if m_name[-11:] == '_embeddings':
pointer = getattr(pointer, 'weight')
elif m_name == 'kernel':
array = np.transpose(array)
try:
assert pointer.shape == array.shape
except AssertionError as e:
e.args += (pointer.shape, array.shape)
raise
pointer.data = torch.from_numpy(array)
# Save pytorch-model
torch.save(model.state_dict(), args.pytorch_dump_path)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_config_file",
default=None,
type=str,
required=True,
help="The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument("--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument("--do_lower_case",
default=False,
action='store_true',
help="Whether to lower case the input text. True for uncased models, False for cased models.")
parser.add_argument("--max_seq_length",
default=128,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion",
default=0.1,
type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumualte before performing a backward/update pass.")
parser.add_argument('--optimize_on_cpu',
default=False,
action='store_true',
help="Whether to perform optimization and keep the optimizer averages on CPU")
parser.add_argument('--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale',
type=float, default=128,
help='Loss scaling, positive power of 2 values can improve fp16 convergence.')
args = parser.parse_args()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
"news": NewsProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
# torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info("16-bits training currently not supported in distributed training")
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format(
args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
label_list = processor.get_labels()
print("label_list.size:%d\n" %(len(label_list)))
# Prepare model
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
model.half()
model.to(device)
#if args.local_rank != -1:
#model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
# output_device=args.local_rank)
#elif n_gpu > 1:
# model = torch.nn.DataParallel(model)
# Prepare optimizer
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = RandomSampler(train_data)
#train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = SequentialSampler(eval_data)
#eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss/nb_tr_steps}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
}
num_labels_task = {
"cola": 2,
"mnli": 3,
"mrpc": 2,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps * args.num_train_epochs)
# Prepare model
model = BertForSequenceClassification.from_pretrained(
args.bert_model,
cache_dir=PYTORCH_PRETRAINED_BERT_CACHE /
'distributed_{}'.format(args.local_rank),
num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
t_total = num_train_steps
if args.local_rank != -1:
t_total = t_total // torch.distributed.get_world_size()
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=t_total)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
# modify learning rate with special warm up BERT uses
lr_this_step = args.learning_rate * warmup_linear(
global_step / t_total, args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")
if args.do_train:
torch.save(model_to_save.state_dict(), output_model_file)
# Load a trained model that you have fine-tuned
model_state_dict = torch.load(output_model_file)
model = BertForSequenceClassification.from_pretrained(
args.bert_model, state_dict=model_state_dict, num_labels=num_labels)
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
default=False,
action='store_true',
help="Whether to run prediction on the test set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument(
'--optimize_on_cpu',
default=False,
action='store_true',
help=
"Whether to perform optimization and keep the optimizer averages on CPU"
)
parser.add_argument(
'--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=128,
help=
'Loss scaling, positive power of 2 values can improve fp16 convergence.'
)
args = parser.parse_args()
processors = {
"dlcompetition": DLCompetitionProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
# torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info(
"16-bits training currently not supported in distributed training"
)
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps * args.num_train_epochs)
label_list = processor.get_labels()
print("label_list.size:%d\n" % (len(label_list)))
# Prepare model
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
model.half()
model.to(device)
# if args.local_rank != -1:
# model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
# output_device=args.local_rank)
# elif n_gpu > 1:
# model = torch.nn.DataParallel(model)
# Prepare optimizer
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [{
'params': [p for n, p in param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = RandomSampler(train_data)
# train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(
param_optimizer,
model.named_parameters(),
test_nan=True)
if is_nan:
logger.info(
"FP16 TRAINING: Nan in gradients, reducing loss scaling"
)
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(
model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = SequentialSampler(eval_data)
# eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_test:
# prediction
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running prediction *****")
logger.info(" Num examples = %d", len(test_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids)
if args.local_rank == -1:
test_sampler = SequentialSampler(test_data)
else:
test_sampler = SequentialSampler(test_data)
# test_sampler = DistributedSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=args.eval_batch_size)
model.eval()
answer = []
for input_ids, input_mask, segment_ids in test_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().tolist()
answer.extend(logits)
output_test_prediction = os.path.join(
args.output_dir,
time.strftime("%Y%m%d-%H%M%S") + "submission.csv")
with open(output_test_prediction, "w") as writer:
logger.info("Writing prediction")
writer.write("id,label\n")
for i, ans in answer:
writer.write("%d,%d\n" % (i, ans))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--train_file",
default=None,
type=str,
required=True,
help="The train file path")
parser.add_argument("--eval_file",
default=None,
type=str,
required=True,
help="The dev file path")
parser.add_argument("--predict_file",
default=None,
type=str,
required=False,
help="The predict file path")
parser.add_argument("--predict_result_file",
default=None,
type=str,
required=False,
help="The predict result file path")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=300,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--num_labels",
default=1,
type=int,
help="mapping classify nums")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=6.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
vocab_path = os.path.join(args.bert_model, VOCAB_NAME)
# bert_config = BertConfig.from_json_file(vocab_path)
data_processor = DataProcessor()
devices = tpu_xm.get_xla_supported_devices()
n_tpu = len(devices)
logging.info(f'Found {n_tpu} TPU cores')
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.do_train:
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(args.output_dir))
else:
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path,
do_lower_case=args.do_lower_case)
model = BertForSequenceClassification.from_pretrained(args.bert_model,
num_labels=3)
for k, v in model.state_dict().items():
print(f'k = {k}, v.grad = {v.grad}')
model = tpu_dp.DataParallel(model, device_ids=devices)
if args.do_train:
# 数据读取
train_examples = data_processor.get_examples(args.train_file,
data_type='train')
eval_examples = data_processor.get_examples(args.eval_file,
data_type='eval')
# 特征转换
train_features = convert_examples_to_features(args, train_examples,
args.max_seq_length,
tokenizer)
eval_features = convert_examples_to_features(args, eval_examples,
args.max_seq_length,
tokenizer)
num_train_steps = int(
len(train_features) // args.train_batch_size //
args.gradient_accumulation_steps * args.num_train_epochs)
# 数据loader
train_loader = ParaDataloader(train_features)
eval_loader = ParaDataloader(eval_features)
# 数据并行loader输入格式
train_loader = DataLoader(train_loader,
shuffle=True,
batch_size=args.train_batch_size)
eval_loader = DataLoader(eval_loader,
shuffle=False,
batch_size=args.eval_batch_size)
def tpu_training_loop(model, loader, device, context):
""" Called by torch_xla_py.data_parallel. This function is executed on each core of the TPU once per epoch"""
model.zero_grad()
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
param_optimizer = list(model.named_parameters())
optimizer_grouped_parameters = [{
'params': [p for n, p in param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = context.getattr_or(
'optimizer',
BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps))
tr_loss = None
pbar = None
if str(pbar_device) == str(device):
pbar = tqdm(total=int(pbar_steps),
desc=f"training",
dynamic_ncols=True)
tracker = tpu_xm.RateTracker()
model.train()
for step, batch in enumerate(loader):
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tracker.add(args.train_batch_size)
tr_loss = loss * args.gradient_accumulation_steps if step == 0 else tr_loss + loss * args.gradient_accumulation_steps
if pbar is not None:
pbar.update(1)
tpu_xm.optimizer_step(optimizer)
# optimizer.step()
optimizer.zero_grad()
return tr_loss.item() / step
def tpu_evaluating_loop(model, eval_dataloader, device, context):
model.eval()
eval_loss = 0
eval_pbar = None
logits, labels = [], []
if str(pbar_device) == str(device):
eval_pbar = tqdm(total=int(eval_pbar_steps),
desc=f"evaluating",
dynamic_ncols=True)
tracker = tpu_xm.RateTracker()
for step, batch in enumerate(eval_dataloader):
input_ids, input_mask, segment_ids, label_ids = batch
with torch.no_grad():
loss, logit = model(input_ids, segment_ids, input_mask,
label_ids)
eval_loss = loss * args.gradient_accumulation_steps if step == 0 else eval_loss + loss * args.gradient_accumulation_steps
logit = torch.argmax(logit, dim=-1)
logits.extend(logit.tolist())
labels.extend(label_ids.tolist())
tracker.add(args.eval_batch_size)
if eval_pbar is not None:
eval_pbar.update(1)
return (eval_loss.item() / step, logits, labels)
def tpu_predicting_loop(model, dataloader, device, context):
model.eval()
eval_pbar = None
logits, example_ids, probs = [], [], []
if str(pbar_device) == str(device):
eval_pbar = tqdm(total=int(eval_pbar_steps),
desc=f"evaluating",
dynamic_ncols=True)
tracker = tpu_xm.RateTracker()
for step, batch in enumerate(dataloader):
input_ids, input_mask, segment_ids, label_ids = batch
with torch.no_grad():
logit = model(input_ids, segment_ids, input_mask)
prob = torch.softmax(logit, dim=-1).tolist()
logit = torch.argmax(logit, dim=-1)
logits.extend(logit.tolist())
example_ids.extend(label_ids.tolist())
probs.extend(prob)
tracker.add(args.eval_batch_size)
if eval_pbar is not None:
eval_pbar.update(1)
return logits, example_ids, probs
def eval_meric(model, loop, data_loader):
eval_results = model(loop, data_loader)
eval_loss, eval_loss = 0, 0
all_logits, all_labels = [], []
assert len(eval_results) == len(devices) == 8
for eval_result in eval_results:
eval_loss += eval_result[0]
all_logits.extend(eval_result[1])
all_labels.extend(eval_result[2])
accuracy(all_labels, all_logits)
logger.info(f'Average eval loss = {eval_loss / len(eval_results)}')
def write_predict_file(model, loop, data_loader, file_path):
"""
写入预测文件： 格式：'五彩滨云-final.csv'
"""
results = model(loop, data_loader)
logits, ids, probs = [], [], []
assert len(results) == len(devices) == 8
for result in results:
logits.extend(result[0])
ids.extend(result[1])
probs.extend(result[2])
assert len(ids) == len(logits)
logger.info(
f'zero nums {logits.count(0)}, one nums {logits.count(1)}, two nums {logits.count(2)}'
)
labels = [
data_processor.eval_dict[id][1] for id, logit in zip(ids, logits)
]
if not args.do_eval:
logits = [i - 1 for i in logits]
data_df = pd.DataFrame({'id': ids, 'y': logits})
data_df1 = pd.DataFrame({'id': ids, 'y': logits, 'probs': probs})
data_df1.to_csv('probs_predict.csv', index=None)
else:
assert len(labels) == len(logits)
accuracy(labels, logits)
passages = [
data_processor.eval_dict[id][0]
for id, logit in zip(ids, logits)
]
assert len(labels) == len(passages)
match_array = np.array((logits)) == np.array(labels)
match_list = match_array.tolist()
data_df = pd.DataFrame({
'id': ids,
'pred': logits,
'real': labels,
'probs': probs,
'match': match_list,
'passage': passages
})
data_df.to_csv(file_path, index=None)
if args.do_train:
for epoch in range(1, int(args.num_train_epochs) + 1, 1):
pbar_device = devices[0]
logger.info(f'Start to evaluate......')
eval_pbar_steps = len(eval_loader) // n_tpu
eval_meric(model, tpu_evaluating_loop, eval_loader)
pbar_steps = len(train_loader) // n_tpu
logging.info(
f'Start training, epoch {epoch} on {len(devices)} cores for {pbar_steps} steps'
)
start = time.time()
losses = model(tpu_training_loop, train_loader)
logging.info(
f'Epoch {epoch} took {round(time.time() - start, 2)} seconds. average train loss: {sum(losses) / len(losses)}'
)
save_checkpoint(model._models[0], epoch, args.output_dir)
logger.info('Train finished......')
elif args.do_predict:
pbar_device = devices[0]
logger.info(f'Start to predict......')
if args.do_eval:
predict_examples = data_processor.get_eval_examples(args.eval_file)
else:
predict_examples = data_processor.get_predict_examples(
args.predict_file)
predict_features = convert_examples_to_features(
args, predict_examples, args.max_seq_length, tokenizer)
predict_loader = ParaDataloader(predict_features)
predict_loader = DataLoader(predict_loader,
shuffle=False,
batch_size=args.eval_batch_size)
eval_pbar_steps = len(predict_loader) // n_tpu
write_predict_file(model, tpu_predicting_loop, predict_loader,
args.predict_result_file)
def main():
bert_path = bp.bert_path
## parameters
do_lower_case = False
do_train = True
do_eval = True
data_dir = '.'
max_seq_length = 128
train_batch_size = 8
eval_batch_size = 8
learning_rate = 2e-5
num_train_epochs = 8.0
seed = 42
output_dir = 'dream_finetuned'
no_cuda = False
bert_config_file = bert_path + 'bert_config.json'
vocab_file = bert_path + 'vocab.txt'
init_checkpoint = bert_path + 'pytorch_model.bin'
gradient_accumulation_steps = 3
warmup_proportion = 0.1
local_rank = -1
# 用来指定GPU
processors = {
"dream": dreamProcessor,
}
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(local_rank != -1))
if gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(gradient_accumulation_steps))
train_batch_size = int(train_batch_size / gradient_accumulation_steps)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
if not do_train and not do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(bert_config_file)
if max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(output_dir) and os.listdir(output_dir):
if do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(output_dir))
else:
os.makedirs(output_dir, exist_ok=True)
processor = processors['dream']()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
train_examples = None
num_train_steps = None
if do_train:
train_examples = processor.get_train_examples(data_dir)
num_train_steps = int(
len(train_examples) / n_class / train_batch_size /
gradient_accumulation_steps * num_train_epochs)
model = BertForSequenceClassification(
bert_config, 1 if n_class > 1 else len(label_list))
if init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(init_checkpoint, map_location='cpu'))
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], output_device=local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=learning_rate,
warmup=warmup_proportion,
t_total=num_train_steps)
global_step = 0
if do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in train_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=train_batch_size)
model.train()
for _ in trange(int(num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids,
n_class)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if gradient_accumulation_steps > 1:
loss = loss / gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), os.path.join(output_dir, "model.pt"))
model.load_state_dict(torch.load(os.path.join(output_dir, "model.pt")))
if do_eval:
eval_examples = processor.get_dev_examples(data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(output_dir, "eval_results_dev.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(output_dir, "logits_dev.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
eval_examples = processor.get_test_examples(data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", eval_batch_size)
input_ids = []
input_mask = []
segment_ids = []
label_id = []
for f in eval_features:
input_ids.append([])
input_mask.append([])
segment_ids.append([])
for i in range(n_class):
input_ids[-1].append(f[i].input_ids)
input_mask[-1].append(f[i].input_mask)
segment_ids[-1].append(f[i].segment_ids)
label_id.append([f[0].label_id])
all_input_ids = torch.tensor(input_ids, dtype=torch.long)
all_input_mask = torch.tensor(input_mask, dtype=torch.long)
all_segment_ids = torch.tensor(segment_ids, dtype=torch.long)
all_label_ids = torch.tensor(label_id, dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids, n_class)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits, label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if do_train:
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
else:
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
output_eval_file = os.path.join(output_dir, "eval_results_test.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(output_dir, "logits_test.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default='/hdd/lujunyu/dataset/multi_turn_corpus/ecommerce/',
type=str,
required=False,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_config_file",
default=BERT_BASE_DIR + 'bert_config.json',
type=str,
required=False,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default='ECD',
type=str,
required=False,
help="The name of the task to train.")
parser.add_argument(
"--vocab_file",
default=BERT_BASE_DIR + 'vocab.txt',
type=str,
required=False,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default='/hdd/lujunyu/model/chatbert/ecd_without_pretraining/',
type=str,
required=False,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=True,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_test",
default=True,
action='store_true',
help="Whether to run eval on the test set.")
parser.add_argument("--train_batch_size",
default=500,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=200,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=1e-3,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=5.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.0,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=200,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=1,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
bert_config.num_hidden_layers = 2
# bert_config.num_attention_heads = 6
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
if args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".
format(args.output_dir))
else:
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese',
do_lower_case=args.do_lower_case)
train_dataset = ECDDatasetForSP(file_path=os.path.join(
args.data_dir, "train.txt"),
max_seq_length=args.max_seq_length,
tokenizer=tokenizer)
eval_dataset = ECDDatasetForSP(file_path=os.path.join(
args.data_dir, "dev.txt"),
max_seq_length=args.max_seq_length,
tokenizer=tokenizer)
train_dataloader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.train_batch_size,
sampler=RandomSampler(train_dataset),
num_workers=4)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=args.eval_batch_size,
sampler=SequentialSampler(eval_dataset),
num_workers=4)
num_train_steps = None
if args.do_train:
num_train_steps = int(
len(train_dataset) / args.train_batch_size /
args.gradient_accumulation_steps * args.num_train_epochs)
model = BertForSequenceClassification(bert_config, num_labels=2)
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# no_decay = ['bias', 'gamma', 'beta']
# optimizer_parameters = [
# {'params': [p for n, p in model.named_parameters() if n not in no_decay], 'weight_decay_rate': 0.01},
# {'params': [p for n, p in model.named_parameters() if n in no_decay], 'weight_decay_rate': 0.0}
# ]
optimizer_parameters = [{
'params': [p for n, p in model.named_parameters()],
'weight_decay_rate': 0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
best_acc = 0.0
if args.do_train:
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
labels=label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if (step + 1) % args.save_checkpoints_steps == 0:
### Evaluate at the end of epoches
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logits_all = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(
input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
labels=label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
for i in range(len(logits)):
logits_all += [logits[i]]
tmp_eval_accuracy = accuracy(logits,
label_ids.reshape(-1))
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy
}
output_eval_file = os.path.join(args.output_dir,
"eval_results_dev.txt")
with open(output_eval_file, "a") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
output_eval_file = os.path.join(args.output_dir,
"logits_dev.txt")
with open(output_eval_file, "w") as f:
for i in range(len(logits_all)):
for j in range(len(logits_all[i])):
f.write(str(logits_all[i][j]))
if j == len(logits_all[i]) - 1:
f.write("\n")
else:
f.write(" ")
### Save the best checkpoint
if best_acc < eval_accuracy:
try: ### Remove 'module' prefix when using DataParallel
state_dict = model.module.state_dict()
except AttributeError:
state_dict = model.state_dict()
torch.save(state_dict,
os.path.join(args.output_dir, "model.pt"))
best_acc = eval_accuracy
logger.info('Saving the best model in {}'.format(
os.path.join(args.output_dir, "model.pt")))
model.train()
def main():
args = get_train_arguments()
device = torch.device(
"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
num_gpu = torch.cuda.device_count()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if num_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
bert_config = BertConfig.from_json_file(args.bert_config_file)
tokenizer = FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
processor = None
if args.task_name == "sentihood_NLI_M":
processor = Sentihood_NLI_M_Processor()
elif args.task_name == "sentihood_NLI_B":
processor = Sentihood_NLI_B_Processor
elif args.task_name == "sentihood_single":
processor = Sentihood_single_Processor()
elif args.task_name == "sentihood_QA_B":
processor = Sentihood_QA_B_Processor()
elif args.task_name == "sentihood_QA_M":
processor = Sentihood_QA_M_Processor()
else:
raise ValueError("Unimplemented task!")
if not os.path.exists(args.output_dir):
print('make output directory {}'.format(args.output_dir))
os.makedirs(args.output_dir)
labels = processor.get_labels()
# training set
if not os.path.exists(args.data_dir):
raise ValueError("Data does not exist")
train_examples = processor.get_train_examples(args.data_dir)
divide = args.train_batch_size * args.num_train_epochs
num_train_steps = int(len(train_examples) / divide)
train_features = get_features(
train_examples, labels, args.max_seq_length, tokenizer)
label_ids, input_ids, seg_ids, input_mks = get_input_tensor(
train_features)
train_dataset = TensorDataset(input_ids, input_mks, seg_ids, label_ids)
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(
train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
# test set
if args.eval_test:
test_examples = processor.get_test_examples(args.data_dir)
test_features = get_features(
test_examples, labels, args.max_seq_length, tokenizer)
label_ids, input_ids, seg_ids, input_mks = get_input_tensor(
test_features)
test_data = TensorDataset(
input_ids, input_mks, seg_ids, label_ids)
test_dataloader = DataLoader(
test_data, batch_size=args.eval_batch_size, shuffle=False)
# model and optimizer
model = BertForSequenceClassification(bert_config, len(labels))
# load the pretrained parameters
model.bert.load_state_dict(torch.load(
args.init_checkpoint, map_location='cpu'))
model.to(device)
if num_gpu > 1:
model = torch.nn.DataParallel(model)
##########################continue here#########################################
with open('{}/log.txt'.format(args.output_dir), "w") as f:
title = "epoch global_step loss\ttest_loss test_accuracy" if args.eval_test else "epoch global_step loss "
f.write(title)
f.write("\n")
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(
nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},
{'params': [p for n, p in model.named_parameters() if any(
nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}
]
# train
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
epoch = 0
for ti in trange(int(args.num_train_epochs), desc="Epoch"):
nb_tr_examples, nb_tr_steps, tr_loss = 0, 0, 0
epoch += 1
model.train()
for i, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
# batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
input_ids, input_mask, segment_ids, label_ids = input_ids.to(
device), input_mask.to(device), segment_ids.to(device), label_ids.to(device)
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if num_gpu > 1:
loss = loss.mean()
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
optimizer.step()
model.zero_grad()
global_step += 1
# eval_test
if args.eval_test:
model.eval()
test_loss, test_accuracy, nb_test_steps, nb_test_examples = 0, 0, 0, 0
fname = "{}/test_ep_{}.txt".format(args.output_dir, epoch)
with open(fname, "w") as ftname:
for batch in test_dataloader:
input_ids, input_mask, segment_ids, label_ids = batch
input_ids, input_mask = input_ids.to(
device), input_mask.to(device),
segment_ids, label_ids = segment_ids.to(
device), label_ids.to(device)
with torch.no_grad():
tmp_test_loss, logits = model(
input_ids, segment_ids, input_mask, label_ids)
label_ids = label_ids.to('cpu').numpy()
logits = F.softmax(logits, dim=-1).detach().cpu().numpy()
outputs = np.argmax(logits, axis=1)
for o_i in range(len(outputs)):
ftname.write(str(outputs[o_i]))
for ou in logits[o_i]:
ftname.write(" " + str(ou))
ftname.write("\n")
test_accuracy += np.sum(outputs == label_ids)
test_loss += tmp_test_loss.mean().item()
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
eval_str = "{} {} {} {}\n".format(epoch, global_step, tr_loss /
nb_tr_steps, test_loss, test_accuracy)
train_str = "{} {} {}\n".format(
epoch, global_step, tr_loss / nb_tr_steps)
row = eval_str if args.eval_test else train_str
with open('{}/log.txt'.format(args.output_dir), "a+") as f:
f.write(row)
def load_model(self, bert_config_file, init_checkpoint, vocab_file,
max_seq_length, task_name, do_lower_case, seed, local_rank,
no_cuda):
logger.info('Calling torch.cuda.current_device [PATCH]')
torch.cuda.current_device()
logger.info(
'[ Local rank: {} ] [ No cuda: {} ] [ Cuda available: {} ]'.format(
local_rank, no_cuda, torch.cuda.is_available()))
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"[ Device: {} ] [ n_gpu: {} ] [ distributed inference: {} ]".
format(device, n_gpu, bool(local_rank != -1)))
logger.info("[ Using seed {} ]".format(seed))
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
logger.info(
"Loading BERT config file from {}".format(bert_config_file))
bert_config = BertConfig.from_json_file(bert_config_file)
if max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(max_seq_length, bert_config.max_position_embeddings))
# prepare dataloaders
processors = {
"sentihood_NLI_B": Sentihood_NLI_B_Processor,
"sentihood_NLI_M": Sentihood_NLI_M_Processor,
}
logger.info("Infering for {} task".format(task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
logger.info(
"Creating tokenizer. [ Vocab file: {} ] [ Lower case: {} ]".format(
vocab_file, do_lower_case))
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=do_lower_case)
# model and optimizer
model = BertForSequenceClassification(bert_config, len(label_list))
model.load_state_dict(torch.load(init_checkpoint, map_location='cpu'))
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], output_device=local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
return model, tokenizer, processor, device
def main():
#rs_writer = SummaryWriter("./log")
parser = argparse.ArgumentParser()
viz = visdom.Visdom()
# Required parameters
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
choices=[
"semeval_QA_EXPT", "semeval_QA_T",
"travel_experience", "semeval_single"
],
help="Name of the task to train")
parser.add_argument("--vocab_file",
default=None,
type=str,
required=True,
help="The path of BERT pre-trained vocab file")
parser.add_argument("--init_checkpoint",
default=None,
type=str,
required=True,
help="The path of BERT pre-trained .ckpt file")
parser.add_argument("--bert_config_file",
default=None,
type=str,
required=True,
help="The path of BERT .json file")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="The output directory of training result")
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The path of training dataset")
# Other parameters
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="The size of training batch")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="The size of evaluation batch")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum sentence length of input after WordPiece tonkenization\n"
"Greater than the max will be truncated, smaller than the max will be padding"
)
parser.add_argument("--local_rank",
default=-1,
type=int,
help="Local_rank for distributed training on gpus")
parser.add_argument("--seed",
default=42,
type=int,
help="Random seed for initialization")
parser.add_argument(
"--accumulate_gradients",
default=1,
type=int,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument(
'--gradient_accumulation_steps',
default=1,
type=int,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
parser.add_argument('--save_steps',
type=int,
default=100,
help="Save checkpoint every X updates steps.")
parser.add_argument(
'--layers',
type=int,
nargs='+',
default=[-2],
help="choose the layers that used for downstream tasks, "
"-2 means use pooled output, -1 means all layer,"
"else means the detail layers. default is -2")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="The number of epochs on training data")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The learning rate of model")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for\n"
"0.1 means 10% of training set")
parser.add_argument('--layer_learning_rate_decay',
type=float,
default=0.95)
parser.add_argument('--layer_learning_rate',
type=float,
nargs='+',
default=[2e-5] * 12,
help="learning rate in each group")
parser.add_argument("--do_train",
default=False,
action="store_true",
help="Whether training the data or not")
parser.add_argument("--do_eval",
default=False,
action="store_true",
help="Whether evaluating the data or not")
parser.add_argument("--do_predict",
default=False,
action="store_true",
help="Whether predicting the data or not")
parser.add_argument(
"--do_lower_case",
default=False,
action="store_true",
help=
"To lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument("--no_cuda",
default=False,
action="store_true",
help="Whether use the GPU device or not")
parser.add_argument("--discr",
default=False,
action='store_true',
help="Whether to do discriminative fine-tuning.")
parser.add_argument('--pooling_type',
default=None,
type=str,
choices=[None, 'mean', 'max'])
args = parser.parse_args()
viz = visdom.Visdom()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
# prepare dataloaders
processors = {
"semeval_QA_EXPT": Semeval_QA_EXPT_Processor,
"semeval_QA_T": Semeval_QA_T_Processor,
"travel_experience": Travel_exp_data,
"semeval_single": Semeval_single_Processor
}
processor = processors[args.task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
# training set
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs)
train_features = convert_examples_to_features(train_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
# test set
if args.do_eval:
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
test_dataloader = DataLoader(test_data,
batch_size=args.eval_batch_size,
shuffle=False)
# model and optimizer layer version
"""
model = BertForSequenceClassification(bert_config, len(label_list), args.layers, pooling=args.pooling_type)
if args.init_checkpoint is not None:
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
if args.discr:
if len(args.layer_learning_rate) > 1:
groups = [(f'layer.{i}.', args.layer_learning_rate[i]) for i in range(12)]
else:
lr = args.layer_learning_rate[0]
groups = [(f'layer.{i}.', lr * pow(args.layer_learning_rate_decay, 11 - i)) for i in range(12)]
group_all = [f'layer.{i}.' for i in range(12)]
no_decay_optimizer_parameters = []
decay_optimizer_parameters = []
for g, l in groups:
no_decay_optimizer_parameters.append(
{
'params': [p for n, p in model.named_parameters() if
not any(nd in n for nd in no_decay) and any(nd in n for nd in [g])],
'weight_decay_rate': 0.01, 'lr': l
}
)
decay_optimizer_parameters.append(
{
'params': [p for n, p in model.named_parameters() if
any(nd in n for nd in no_decay) and any(nd in n for nd in [g])],
'weight_decay_rate': 0.0, 'lr': l
}
)
group_all_parameters = [
{'params': [p for n, p in model.named_parameters() if
not any(nd in n for nd in no_decay) and not any(nd in n for nd in group_all)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in model.named_parameters() if
any(nd in n for nd in no_decay) and not any(nd in n for nd in group_all)],
'weight_decay_rate': 0.0},
]
optimizer_parameters = no_decay_optimizer_parameters + decay_optimizer_parameters + group_all_parameters
else:
optimizer_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
"""
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.01
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
# train
output_log_file = os.path.join(args.output_dir, "log.txt")
print("output_log_file=", output_log_file)
with open(output_log_file, "w") as writer:
if args.do_eval:
writer.write(
"epoch\tglobal_step\tloss\ttest_loss\ttest_accuracy\n")
else:
writer.write("epoch\tglobal_step\tloss\n")
global_step = 0
epoch = 0
best_epoch, best_accuracy = 0, 0
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
epoch += 1
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
print(loss.item())
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
viz.line([loss.item()], [global_step],
win='tr_loss',
update='append')
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
# Save the checkpoint model after each N steps
if args.local_rank in [
-1, 0
] and args.save_steps > 0 and global_step % args.save_steps == 0:
save_output_dir = os.path.join(
args.output_dir, "checkpoint-{}".format(global_step))
if not os.path.exists(save_output_dir):
os.makedirs(save_output_dir)
torch.save(
model.state_dict(),
os.path.join(save_output_dir, "training_args.bin"))
viz.line([optimizer.get_lr()[0]], [global_step - 1],
win="lr",
update="append")
# eval_test
if args.do_eval:
model.eval()
test_loss, test_accuracy = 0, 0
nb_test_steps, nb_test_examples = 0, 0
with open(
os.path.join(args.output_dir,
"test_ep_" + str(epoch) + ".txt"),
"w") as f_test:
for input_ids, input_mask, segment_ids, label_ids in tqdm(
test_dataloader, desc="Testing"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_test_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids)
logits = F.softmax(logits, dim=-1)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
outputs = np.argmax(logits, axis=1)
for output_i in range(len(outputs)):
f_test.write(str(outputs[output_i]))
for ou in logits[output_i]:
f_test.write(" " + str(ou))
f_test.write("\n")
tmp_test_accuracy = np.sum(outputs == label_ids)
viz.line([tmp_test_loss.item()], [nb_test_steps],
win='eval_loss',
update='append')
test_loss += tmp_test_loss.mean().item()
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
nb_test_steps += 1
test_loss = test_loss / nb_test_steps
test_accuracy = test_accuracy / nb_test_examples
viz.line([test_accuracy], [nb_test_steps - 1],
win='test_acc',
update='append')
if test_accuracy > best_accuracy:
best_accuracy = test_accuracy
best_epoch = epoch
torch.save(model.state_dict(),
os.path.join(args.output_dir, "best_model.bin"))
result = collections.OrderedDict()
if args.do_eval:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps,
'test_loss': test_loss,
'test_accuracy': test_accuracy
}
else:
result = {
'epoch': epoch,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
logger.info("***** Eval results *****")
with open(output_log_file, "a+") as writer:
for key in result.keys():
logger.info(" %s = %s\n", key, str(result[key]))
writer.write("%s\t" % (str(result[key])))
writer.write("\n")
print("The best Epoch is: ", best_epoch)
print("The best test_accuracy is: ", best_accuracy)
class Instructor:
def __init__(self, args):
self.opt = args
self.writer = SummaryWriter(log_dir=self.opt.output_dir) # tensorboard
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length,
bert_config.max_position_embeddings))
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
# os.makedirs(args.output_dir, exist_ok=True)
self.dataset = ReadData(self.opt) # Read the data and preprocess it
self.num_train_steps = None
self.num_train_steps = int(
len(self.dataset.train_examples) / self.opt.train_batch_size /
self.opt.gradient_accumulation_steps * self.opt.num_train_epochs)
self.opt.label_size = len(self.dataset.label_list)
args.output_dim = len(self.dataset.label_list)
print("label size: {}".format(args.output_dim))
# 初始化模型
print("initialize model ...")
if args.model_class == BertForSequenceClassification:
self.model = BertForSequenceClassification(
bert_config, len(self.dataset.label_list))
else:
self.model = model_classes[args.model_name](bert_config, args)
if args.init_checkpoint is not None:
self.model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
self.model.half()
# 冻结参数
# for name, p in self.model.named_parameters():
# if name.startswith('bert.encoder.layer.11') or name.startswith('bert.encoder.layer.10') or name.startswith('bert.encoder.layer.9') or name.startswith('bert.encoder.layer.8'): # 冻结最后一层
# p.requires_grad = False
# 计算模型的参数个数
n_trainable_params, n_nontrainable_params = 0, 0
for p in self.model.parameters():
n_params = torch.prod(torch.tensor(
p.shape)) # torch.prod()表示计算所有元素的乘积
if p.requires_grad: # 是否需要求梯度
n_trainable_params += n_params
else:
n_nontrainable_params += n_params
print('n_trainable_params: {0}, n_nontrainable_params: {1}'.format(
n_trainable_params, n_nontrainable_params))
self.model.to(args.device)
# 并行化
if self.opt.n_gpu > 1:
self.model = torch.nn.DataParallel(self.model,
device_ids=self.opt.gpu_id)
# Prepare optimizer
if args.fp16:
self.param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in self.model.named_parameters()]
elif args.optimize_on_cpu:
self.param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in self.model.named_parameters()]
else:
self.param_optimizer = list(self.model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [{
'params':
[p for n, p in self.param_optimizer if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in self.param_optimizer if n in no_decay],
'weight_decay_rate':
0.0
}]
self.optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=self.num_train_steps)
# 配置自己模型的优化器
# [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')]
# self.optimizer_me = torch.optim.Adam(
# [{'params': [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')]}], lr=0.001,
# weight_decay=0)
# self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer_me, mode='max',
# patience=3) # 3个epoch后，所监测的值停止增加时自动调整学习率
self.global_step = 0 # 初始化全局步数为 0
self.max_test_acc = 0
self.max_test_f1 = 0
def do_train(self): # 训练模型
# for _ in trange(int(args.num_train_epochs), desc="Epoch"):
for i_epoch in range(int(args.num_train_epochs)):
print('>' * 100)
print('epoch: ', i_epoch)
tr_loss = 0
train_accuracy = 0
nb_tr_examples, nb_tr_steps = 0, 0
y_pred = []
y_true = []
self.model.train() # 让模型处于训练状态，因为每跑完一个epoch就会处于测试状态
for step, batch in enumerate(
tqdm(self.dataset.train_dataloader, desc="Training")):
# batch = tuple(t.to(self.opt.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids, \
input_t_ids, input_t_mask, segment_t_ids, \
input_without_t_ids, input_without_t_mask, segment_without_t_ids, \
input_left_t_ids, input_left_t_mask, segment_left_t_ids, \
input_right_t_ids, input_right_t_mask, segment_right_t_ids, \
input_left_ids, input_left_mask, segment_left_ids = batch
input_ids = input_ids.to(self.opt.device)
segment_ids = segment_ids.to(self.opt.device)
input_mask = input_mask.to(self.opt.device)
label_ids = label_ids.to(self.opt.device)
if self.opt.model_class in [
BertForSequenceClassification, CNN
]:
loss, logits = self.model(input_ids, segment_ids,
input_mask, label_ids)
else:
input_t_ids = input_t_ids.to(self.opt.device)
input_t_mask = input_t_mask.to(self.opt.device)
segment_t_ids = segment_t_ids.to(self.opt.device)
if self.opt.model_class == MemNet:
input_without_t_ids = input_without_t_ids.to(
self.opt.device)
input_without_t_mask = input_without_t_mask.to(
self.opt.device)
segment_without_t_ids = segment_without_t_ids.to(
self.opt.device)
loss, logits = self.model(input_without_t_ids,
segment_without_t_ids,
input_without_t_mask,
label_ids, input_t_ids,
input_t_mask, segment_t_ids)
elif self.opt.model_class in [Cabasc]:
input_left_t_ids = input_left_t_ids.to(self.opt.device)
input_left_t_mask = input_left_t_mask.to(
self.opt.device)
segment_left_t_ids = segment_left_t_ids.to(
self.opt.device)
input_right_t_ids = input_right_t_ids.to(
self.opt.device)
input_right_t_mask = input_right_t_mask.to(
self.opt.device)
segment_right_t_ids = segment_right_t_ids.to(
self.opt.device)
loss, logits = self.model(
input_ids, segment_ids, input_mask, label_ids,
input_t_ids, input_t_mask, segment_t_ids,
input_left_t_ids, input_left_t_mask,
segment_left_t_ids, input_right_t_ids,
input_right_t_mask, segment_right_t_ids)
elif self.opt.model_class in [
RAM, TNet_LF, MGAN, TT, MLP, TD_BERT, TD_BERT_QA,
DTD_BERT
]:
input_left_ids = input_left_ids.to(self.opt.device)
input_left_mask = input_left_mask.to(self.opt.device)
segment_left_ids = segment_left_ids.to(self.opt.device)
loss, logits = self.model(
input_ids, segment_ids, input_mask, label_ids,
input_t_ids, input_t_mask, segment_t_ids,
input_left_ids, input_left_mask, segment_left_ids)
else:
loss, logits = self.model(input_ids, segment_ids,
input_mask, label_ids,
input_t_ids, input_t_mask,
segment_t_ids)
if self.opt.n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
tr_loss += loss.item()
loss.backward()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
# 计算准确率
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_train_accuracy = accuracy(logits, label_ids)
y_pred.extend(np.argmax(logits, axis=1))
y_true.extend(label_ids)
train_accuracy += tmp_train_accuracy
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in self.model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(
self.param_optimizer,
self.model.named_parameters(),
test_nan=True)
if is_nan:
logger.info(
"FP16 TRAINING: Nan in gradients, reducing loss scaling"
)
args.loss_scale = args.loss_scale / 2
self.model.zero_grad()
continue
self.optimizer.step()
# self.optimizer_me.step()
copy_optimizer_params_to_model(
self.model.named_parameters(),
self.param_optimizer)
else:
self.optimizer.step()
# self.optimizer_me.step()
self.model.zero_grad()
self.global_step += 1
train_accuracy = train_accuracy / nb_tr_examples
train_f1 = f1_score(y_true,
y_pred,
average='macro',
labels=np.unique(y_true))
result = self.do_eval() # 每跑完一轮，测试一次
tr_loss = tr_loss / nb_tr_steps
# self.scheduler.step(result['eval_accuracy']) # 监测验证集的精度
self.writer.add_scalar('train_loss', tr_loss, i_epoch)
self.writer.add_scalar('train_accuracy', train_accuracy, i_epoch)
self.writer.add_scalar('eval_accuracy', result['eval_accuracy'],
i_epoch)
self.writer.add_scalar('eval_loss', result['eval_loss'], i_epoch)
# self.writer.add_scalar('lr', self.optimizer_me.param_groups[0]['lr'], i_epoch)
print(
"Results: train_acc: {0:.6f} | train_f1: {1:.6f} | train_loss: {2:.6f} | eval_accuracy: {3:.6f} | eval_loss: {4:.6f} | eval_f1: {5:.6f} | max_test_acc: {6:.6f} | max_test_f1: {7:.6f}"
.format(train_accuracy, train_f1, tr_loss,
result['eval_accuracy'], result['eval_loss'],
result['eval_f1'], self.max_test_acc,
self.max_test_f1))
def do_eval(self): # 测试准确率
self.model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
# confidence = []
y_pred = []
y_true = []
for batch in tqdm(self.dataset.eval_dataloader, desc="Evaluating"):
# batch = tuple(t.to(self.opt.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids, \
input_t_ids, input_t_mask, segment_t_ids, \
input_without_t_ids, input_without_t_mask, segment_without_t_ids, \
input_left_t_ids, input_left_t_mask, segment_left_t_ids, \
input_right_t_ids, input_right_t_mask, segment_right_t_ids, \
input_left_ids, input_left_mask, segment_left_ids = batch
input_ids = input_ids.to(self.opt.device)
segment_ids = segment_ids.to(self.opt.device)
input_mask = input_mask.to(self.opt.device)
label_ids = label_ids.to(self.opt.device)
with torch.no_grad(): # 不计算梯度
if self.opt.model_class in [
BertForSequenceClassification, CNN
]:
loss, logits = self.model(input_ids, segment_ids,
input_mask, label_ids)
else:
input_t_ids = input_t_ids.to(self.opt.device)
input_t_mask = input_t_mask.to(self.opt.device)
segment_t_ids = segment_t_ids.to(self.opt.device)
if self.opt.model_class == MemNet:
input_without_t_ids = input_without_t_ids.to(
self.opt.device)
input_without_t_mask = input_without_t_mask.to(
self.opt.device)
segment_without_t_ids = segment_without_t_ids.to(
self.opt.device)
loss, logits = self.model(input_without_t_ids,
segment_without_t_ids,
input_without_t_mask,
label_ids, input_t_ids,
input_t_mask, segment_t_ids)
elif self.opt.model_class in [Cabasc]:
input_left_t_ids = input_left_t_ids.to(self.opt.device)
input_left_t_ids = input_left_t_ids.to(self.opt.device)
input_left_t_mask = input_left_t_mask.to(
self.opt.device)
segment_left_t_ids = segment_left_t_ids.to(
self.opt.device)
input_right_t_ids = input_right_t_ids.to(
self.opt.device)
input_right_t_mask = input_right_t_mask.to(
self.opt.device)
segment_right_t_ids = segment_right_t_ids.to(
self.opt.device)
loss, logits = self.model(
input_ids, segment_ids, input_mask, label_ids,
input_t_ids, input_t_mask, segment_t_ids,
input_left_t_ids, input_left_t_mask,
segment_left_t_ids, input_right_t_ids,
input_right_t_mask, segment_right_t_ids)
elif self.opt.model_class in [
RAM, TNet_LF, MGAN, TT, MLP, TD_BERT, TD_BERT_QA,
DTD_BERT
]:
input_left_ids = input_left_ids.to(self.opt.device)
input_left_mask = input_left_mask.to(self.opt.device)
segment_left_ids = segment_left_ids.to(self.opt.device)
loss, logits = self.model(
input_ids, segment_ids, input_mask, label_ids,
input_t_ids, input_t_mask, segment_t_ids,
input_left_ids, input_left_mask, segment_left_ids)
else:
loss, logits = self.model(input_ids, segment_ids,
input_mask, label_ids,
input_t_ids, input_t_mask,
segment_t_ids)
# with torch.no_grad(): # 不计算梯度
# if self.opt.model_class in [BertForSequenceClassification, CNN]:
# loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids)
# else:
# loss, logits = self.model(input_ids, segment_ids, input_mask, labels=label_ids,
# input_t_ids=input_t_ids,
# input_t_mask=input_t_mask, segment_t_ids=segment_t_ids)
# confidence.extend(torch.nn.Softmax(dim=1)(logits)[:, 1].tolist()) # 获取 positive 类的置信度
# loss = F.cross_entropy(logits, label_ids, size_average=False) # 计算mini-batch的loss总和
if self.opt.n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
y_pred.extend(np.argmax(logits, axis=1))
y_true.extend(label_ids)
# eval_loss += tmp_eval_loss.mean().item()
eval_loss += loss.item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
# eval_loss = eval_loss / len(self.dataset.eval_examples)
test_f1 = f1_score(y_true,
y_pred,
average='macro',
labels=np.unique(y_true))
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
if eval_accuracy > self.max_test_acc:
self.max_test_acc = eval_accuracy
if self.opt.do_predict: # 测试模式才保存模型
torch.save(self.model, self.opt.model_save_path)
if test_f1 > self.max_test_f1:
self.max_test_f1 = test_f1
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'eval_f1': test_f1,
}
# output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
# with open(output_eval_file, "w") as writer:
# logger.info("***** Eval results *****")
# for key in sorted(result.keys()):
# logger.info(" %s = %s", key, str(result[key]))
# writer.write("%s = %s\n" % (key, str(result[key])))
# print("Eval results ==> eval_accuracy: {0}, eval_loss: {1}, max_test_acc: {2}".format(
# result['eval_accuracy'], result['eval_loss'], self.max_test_acc))
return result
def do_predict(self):
# 加载保存的模型进行预测，获得准确率
# 读测试集的数据
# dataset = ReadData(self.opt) # 这个方法有点冗余了，读取了所有的数据，包括训练集
# Load model
saved_model = torch.load(self.opt.model_save_path)
saved_model.to(self.opt.device)
saved_model.eval()
nb_test_examples = 0
test_accuracy = 0
for batch in tqdm(self.dataset.eval_dataloader, desc="Testing"):
batch = tuple(t.to(self.opt.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids, input_t_ids, input_t_mask, segment_t_ids = batch
with torch.no_grad(): # Do not calculate gradient
if self.opt.model_class in [
BertForSequenceClassification, CNN
]:
_, logits = saved_model(input_ids, segment_ids, input_mask,
label_ids)
else:
_, logits = saved_model(input_ids,
segment_ids,
input_mask,
labels=label_ids,
input_t_ids=input_t_ids,
input_t_mask=input_t_mask,
segment_t_ids=segment_t_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_test_accuracy = accuracy(logits, label_ids)
test_accuracy += tmp_test_accuracy
nb_test_examples += input_ids.size(0)
test_accuracy = test_accuracy / nb_test_examples
return test_accuracy
def run(self):
print('> training arguments:')
for arg in vars(self.opt):
print('>>> {0}: {1}'.format(arg, getattr(self.opt, arg)))
self.do_train()
print('>' * 100)
if self.opt.do_predict:
test_accuracy = self.do_predict()
print("Test Set Accuracy: {}".format(test_accuracy))
print("Max validate Set Acc: {0}".format(
self.max_test_acc)) # Output the final test accuracy
self.writer.close()
# return self.max_test_acc
return self.max_test_f1
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default='twitter',
type=str,
required=False,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default="bert-base-uncased",
type=str,
required=False,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--input_processor_type",
default="sst-wiki",
type=str,
required=False,
help="The type of processor to use for reading data.")
parser.add_argument(
"--output_dir",
default="output/twitter/",
type=str,
required=False,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=True,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--hammer_coeff',
type=float,
default=0.0,
help="Hammer loss coefficient")
parser.add_argument('--att_opt_func',
type=str,
default="mean",
help="Attention optimization function")
parser.add_argument("--debug", action='store_true')
parser.add_argument("--first_run", action='store_true')
parser.add_argument("--name", type=str)
# argv = ['data/twitter', "bert-base-uncased", "sst-wiki", "output/twitter/"]
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
base_labels = {}
print(f"FIRST RUN: {args.first_run}")
# if not args.first_run:
# for typ in ["dev", "test"]:
# base_labels_content = open("{}_base_labels_{}.txt".format(args.name, typ), 'r').readlines()
# base_labels[typ] = [int(label.strip()) for label in base_labels_content]
#
debug = args.debug
if debug:
args.train_batch_size = 2
args.eval_batch_size = 2
args.num_train_epochs = 1
processors = {"sst-wiki": SstWikiProcessor, "pronoun": PronounProcessor}
output_modes = {
"sst-wiki": "classification",
"pronoun": "classification",
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
input_processor_type = args.input_processor_type.lower()
if input_processor_type not in processors:
raise ValueError("Task not found: %s" % (input_processor_type))
processor = processors[input_processor_type]()
output_mode = output_modes[input_processor_type]
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.bert_model,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
limit = 2 if debug else 0
train_examples = processor.get_train_examples(args.data_dir, limit)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
warmup_linear = WarmupLinearSchedule(
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
print(
"typ\tepoch\tacc\tavg_mean_mass\tavg_max_mass\tloss\thammer_loss\tlabel_match_score\tavg_mean_vn\tavg_max_vn\tavg_min_vn"
)
model.train()
for epoch in trange(int(args.num_train_epochs) + 1, desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
if epoch > 0:
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
# define a new function to compute loss values for both output_modes
logits, attention_probs_layers, category_mask, _ = model(
input_ids,
token_type_ids=segment_ids,
pad_attention_mask=input_mask,
manipulate_attention=True,
category_mask=None,
labels=None)
# logits - B x 2
loss_fct = CrossEntropyLoss() # averages the loss over B
loss = loss_fct(logits,
torch.argmax(label_ids.view(-1), 1).long())
# loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
loss += attention_regularization_loss(
attention_probs_layers,
category_mask,
input_mask,
args.hammer_coeff,
optimize_func=args.att_opt_func,
debug=debug)[0]
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear.get_lr(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
if debug:
break
# EVALUATION AFTER EVERY EPOCH
eval_preds = {}
for typ in ["dev", "test"]:
eval_preds[typ] = run_evaluation(args, processor, label_list,
tokenizer, output_mode, epoch,
model, num_labels, tr_loss,
global_step, device,
input_processor_type,
base_labels, debug, typ)
#dump labels after the last epoch, or when first_run
if args.first_run or epoch == args.num_train_epochs:
for typ in ["dev", "test"]:
preds = eval_preds[typ]
filename = "{}_labels_{}_.txt".format(typ, epoch)
labels_file = os.path.join(args.output_dir, filename)
with open(labels_file, "w") as writer:
logger.info("Dumping labels in the file: {}".format(
labels_file))
writer.write('\n'.join([str(pred) for pred in preds]))
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Save a trained model, configuration and tokenizer
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
# Load a trained model and vocabulary that you have fine-tuned
model = BertForSequenceClassification.from_pretrained(
args.output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
else:
model = BertForSequenceClassification.from_pretrained(
args.bert_model, num_labels=num_labels)
model.to(device)
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--train_file",
default=None,
type=str,
required=True,
help="The train file path")
parser.add_argument("--eval_file",
default=None,
type=str,
required=True,
help="The dev file path")
parser.add_argument("--eval_train_file",
default=None,
type=str,
required=True,
help="The train eval file path")
parser.add_argument("--predict_file",
default=None,
type=str,
required=False,
help="The predict file path")
parser.add_argument("--top_n",
default=5,
type=float,
required=True,
help="higher than threshold is classify 1,")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
required=False,
help="adam eplisp")
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument(
"--result_file",
default=None,
type=str,
required=False,
help="The result file that the BERT model was trained on.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
# Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument("--do_lower_case",
default=False,
action='store_true',
help="Whether to lower case the input text.")
parser.add_argument(
"--max_seq_length",
default=280,
type=int,
help="maximum total input sequence length after WordPiece tokenization."
)
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_predict",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--num_labels",
default=1,
type=int,
help="mapping classify nums")
parser.add_argument("--reduce_dim",
default=64,
type=int,
help="from hidden size to this dimensions, reduce dim")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=6.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--weight_decay",
default=0.01,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumualte before")
parser.add_argument(
'--optimize_on_cpu',
default=False,
action='store_true',
help="Whether to perform optimization and averages on CPU")
parser.add_argument(
'--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=128,
help='Loss scale, positive power of 2 can improve fp16 convergence.')
args = parser.parse_args()
data_processor = DataProcessor(args.num_labels)
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info(
"16-bits training currently not supported in distributed training"
)
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size /
args.gradient_accumulation_steps)
print(f'args.train_batch_size = {args.train_batch_size}')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
# bert_config.reduce_dim = args.reduce_dim
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if args.do_train:
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
def prepare_data(args, task_name='train'):
if task_name == 'train':
file_path = args.train_file
elif task_name == 'eval':
file_path = args.eval_file
elif task_name == 'train_eval':
file_path = args.eval_train_file
elif task_name == 'predict':
file_path = args.predict_file
if os.path.isdir(file_path):
examples = data_processor.read_file_dir(file_path,
top_n=args.top_n)
else:
if task_name == 'predict':
examples, example_map_ids = data_processor.read_predict_examples(
file_path, top_n=args.top_n, task_name=task_name)
else:
examples, example_map_ids = data_processor.read_novel_examples(
file_path, top_n=args.top_n, task_name=task_name)
features = convert_examples_to_features(examples, args.max_seq_length,
tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features],
dtype=torch.long)
all_example_ids = torch.tensor([f.example_id for f in features],
dtype=torch.long)
if task_name in ['train', 'eval', 'train_eval', 'predict']:
all_label_ids = torch.tensor([f.label_id for f in features],
dtype=torch.long)
datas = TensorDataset(all_example_ids, all_input_ids,
all_input_mask, all_segment_ids,
all_label_ids)
else:
datas = TensorDataset(all_example_ids, all_input_ids,
all_input_mask, all_segment_ids)
if task_name == 'train':
if args.local_rank == -1:
data_sampler = RandomSampler(datas)
else:
data_sampler = DistributedSampler(datas)
dataloader = DataLoader(datas,
sampler=data_sampler,
batch_size=args.train_batch_size,
drop_last=True)
else:
dataloader = DataLoader(datas,
batch_size=args.eval_batch_size,
drop_last=True)
return (dataloader,
example_map_ids) if task_name != 'train' else dataloader
def accuracy(example_ids, logits, probs=None, data_type='eval'):
logits = logits.tolist()
example_ids = example_ids.tolist()
assert len(logits) == len(example_ids)
classify_name = ['no_answer', 'yes_answer']
labels, text_a, novel_names = [], [], []
map_dicts = example_map_ids if data_type == 'eval' else train_example_map_ids
for i in example_ids:
example = map_dicts[i]
labels.append(example.label)
text_a.append("||".join(example.text_a))
novel_names.append(example.name)
write_data = pd.DataFrame({
"text_a": text_a,
"labels": labels,
"logits": logits,
"novel_names": novel_names,
})
write_data['yes_or_no'] = write_data['labels'] == write_data['logits']
if probs is not None:
write_data['logits'] = probs.tolist()
write_data.to_csv(args.result_file, index=False)
assert len(labels) == len(logits)
result = classification_report(labels,
logits,
target_names=classify_name)
return result
def eval_model(model, eval_dataloader, device, data_type='eval'):
model.eval()
eval_loss = 0
all_logits = []
all_example_ids = []
all_probs = []
accuracy_result = None
batch_count = 0
for step, batch in enumerate(tqdm(eval_dataloader, desc="evaluating")):
example_ids, input_ids, input_mask, segment_ids, label_ids = batch
label_ids = label_ids.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
if not args.do_train:
label_ids = None
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids,
segment_ids,
input_mask,
labels=label_ids)
argmax_logits = torch.argmax(logits, dim=1)
first_indices = torch.arange(argmax_logits.size()[0])
logits_probs = logits[first_indices, argmax_logits]
if args.do_train:
eval_loss += tmp_eval_loss.mean().item()
all_logits.append(argmax_logits)
all_example_ids.append(example_ids)
else:
all_logits.append(argmax_logits)
all_example_ids.append(example_ids)
all_probs.append(logits_probs)
batch_count += 1
if all_logits:
all_logits = torch.cat(all_logits, dim=0)
all_example_ids = torch.cat(all_example_ids, dim=0)
all_probs = torch.cat(all_probs, dim=0) if len(all_probs) else None
accuracy_result = accuracy(all_example_ids,
all_logits,
probs=all_probs,
data_type=data_type)
eval_loss /= batch_count
print(f'\n========= {data_type} acc ============')
print(f'{accuracy_result}')
return eval_loss, accuracy_result, all_logits
def predict_model(model, predict_dataloader, device):
model.eval()
all_label_ids = []
all_probs = []
scene_cut_indexs = []
is_cut_probs = []
for step, batch in enumerate(
tqdm(predict_dataloader, desc="predicting")):
example_ids, input_ids, input_mask, segment_ids, label_ids = batch
# print(f'input_ids = {input_ids.shape}')
# print(f'input_mask = {input_mask.shape}')
# print(f'segment_ids = {segment_ids.shape}')
# print(f'label_ids = {label_ids.shape}')
with torch.no_grad():
_, logits = model(input_ids,
segment_ids,
input_mask,
labels=None)
argmax_logits = torch.argmax(logits, dim=1)
first_indices = torch.arange(argmax_logits.size()[0])
logits_probs = logits[first_indices, argmax_logits]
all_label_ids.extend(label_ids.tolist())
all_probs.extend(logits_probs.tolist())
for label, prob, pred_label in zip(label_ids.tolist(),
logits_probs.tolist(),
argmax_logits.tolist()):
if pred_label:
scene_cut_indexs.append(label)
is_cut_probs.append(prob)
# print(f'prob = {prob}')
# print(f'pred label = {pred_label}')
# print(f'sent = {predict_example_map_ids[label]}')
print(f'all_example_ids = {len(all_label_ids)}')
print(f'all_probs = {len(all_probs)}')
assert len(all_label_ids) == len(all_probs)
return scene_cut_indexs, is_cut_probs
if args.do_train:
train_dataloader = prepare_data(args, task_name='train')
num_train_steps = int(
len(train_dataloader) / args.gradient_accumulation_steps *
args.num_train_epochs)
model = BertForSequenceClassification(bert_config,
num_labels=data_processor.num_labels)
new_state_dict = model.state_dict()
init_state_dict = torch.load(
os.path.join(args.bert_model, 'pytorch_model.bin'))
for k, v in init_state_dict.items():
if k in new_state_dict:
print(f'k in = {k} v in shape = {v.shape}')
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
init_model_token_type(model, type_vocab_size=args.top_n)
for k, v in model.state_dict().items():
print(f'k = {k}, v shape {v.shape}')
if args.fp16:
model.half()
if args.do_predict:
model_path = os.path.join(args.output_dir, WEIGHTS_NAME)
new_state_dict = torch.load(model_path)
new_state_dict = dict([(k[7:], v) if k.startswith('module') else (k, v)
for k, v in new_state_dict.items()])
model.load_state_dict(new_state_dict)
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1 and not device == 'cpu':
model = torch.nn.DataParallel(model)
# # model = BalancedDataParallel(1, model, dim=0).to(device)
eval_dataloader, example_map_ids = prepare_data(args, task_name='eval')
train_eval_dataloader, train_example_map_ids = prepare_data(
args, task_name='train_eval')
def get_linear_schedule_with_warmup(optimizer,
num_warmup_steps,
num_training_steps,
last_epoch=-1):
""" Create a schedule with a learning rate that decreases linearly after
linearly increasing during a warmup period.
"""
def lr_lambda(current_step):
if current_step < num_warmup_steps:
return float(current_step) / float(max(1, num_warmup_steps))
return max(
0.0,
float(num_training_steps - current_step) /
float(max(1, num_training_steps - num_warmup_steps)))
return LambdaLR(optimizer, lr_lambda, last_epoch)
def get_optimizers(num_training_steps: int, model):
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
args.weight_decay,
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
warmup_steps = args.warmup_proportion * num_training_steps
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=warmup_steps,
num_training_steps=num_training_steps)
return optimizer, scheduler
if args.do_train:
optimizer, scheduler = get_optimizers(
num_training_steps=num_train_steps, model=model)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
# eval_loss, acc, _ = eval_model(model, eval_dataloader, device)
# logger.info(f'初始开发集loss: {eval_loss}')
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
torch.cuda.empty_cache()
model_save_path = os.path.join(args.output_dir,
f"{WEIGHTS_NAME}.{epoch}")
tr_loss = 0
train_batch_count = 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="training")):
_, input_ids, input_mask, segment_ids, label_ids = batch
label_ids = label_ids.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
loss, _ = model(input_ids,
segment_ids,
input_mask,
labels=label_ids)
if n_gpu > 1:
loss = loss.mean()
if args.fp16 and args.loss_scale != 1.0:
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step()
model.zero_grad()
train_batch_count += 1
tr_loss /= train_batch_count
eval_loss, acc, _ = eval_model(model, eval_dataloader, device)
eval_model(model,
train_eval_dataloader,
device,
data_type='train_eval')
logger.info(
f'训练loss: {tr_loss}, 开发集loss：{eval_loss} 训练轮数：{epoch + 1}/{int(args.num_train_epochs)}'
)
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model.state_dict(), model_save_path)
if epoch == 0:
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
if args.do_predict:
# eval_model(model, train_eval_dataloader, device, data_type='train_eval')
# for root, dir_, files in os.walk(''):
# for file in files:
# arg.predict_file = os.path.join(root, file)
predict_dataloader, predict_example_map_ids = prepare_data(
args, task_name='predict')
scene_cut_indexs, is_cut_probs = predict_model(model,
predict_dataloader,
device)
assert len(scene_cut_indexs) == len(is_cut_probs)
predict_novel_name = os.path.join(
os.path.split(args.predict_file)[0], 'scene_cut_datas',
os.path.split(args.predict_file)[-1])
for i, index in enumerate(scene_cut_indexs):
predict_example_map_ids[
index] = f'######## {round(is_cut_probs[i], 3)} {predict_example_map_ids[index]}'
with open(predict_novel_name, 'w', encoding='utf-8') as f:
for i, line in enumerate(predict_example_map_ids):
f.write(line)
def load_model():
task_name = "tweetmn"
# data_dir =
bert_model = "./models/uncased_bert_base_pytorch"
output_dir = "./output/tweetmn-epoch-10/"
# Other parameters
cache_dir = ""
max_seq_length = 128
do_train = True
do_lower_case = True
eval_batch_size = 8
learning_rate = 5e-5
num_train_epochs = 3.0
warmup_proportion = 0.1
no_cuda = True
overwrite_output_dir = True
local_rank = -1
seed = 42
gradient_accumulation_steps = 1
fp16 = True
loss_scale = 0
server_ip = ''
server_port = ''
if server_ip and server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(server_ip, server_port),
redirect_output=True)
ptvsd.wait_for_attach()
if local_rank == -1 or no_cuda:
device = torch.device(
"cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(local_rank)
device = torch.device("cuda", local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
device = device
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if local_rank in [-1, 0] else logging.WARN)
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(local_rank != -1), fp16))
if gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(argsgradient_accumulation_steps))
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
if not os.path.exists(output_dir) and local_rank in [-1, 0]:
os.makedirs(output_dir)
task_name = task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
if local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
# tokenizer = BertTokenizer.from_pretrained(bert_model, do_lower_case=do_lower_case)
# model = BertForSequenceClassification.from_pretrained(bert_model, num_labels=num_labels)
model = BertForSequenceClassification.from_pretrained(
output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(bert_model,
do_lower_case=do_lower_case)
if local_rank == 0:
torch.distributed.barrier()
# if fp16:
# model.half()
model.to(device)
if local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[local_rank],
output_device=local_rank,
find_unused_parameters=True)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
# Load a trained model and vocabulary that you have fine-tuned
# model = BertForSequenceClassification.from_pretrained(bert_model, num_labels=num_labels)
model.to(device)
return model, tokenizer, processor, output_mode, device
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir", default=None, type=str, required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_config_file", default=None, type=str, required=True,
help="The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--vocab_file", default=None, type=str, required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument("--output_dir", default=None, type=str, required=True,
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--init_checkpoint", default=None, type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument("--do_lower_case", default=False, action='store_true',
help="Whether to lower case the input text. True for uncased models, False for cased models.")
parser.add_argument("--max_seq_length", default=128, type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train", default=False, action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval", default=False, action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size", default=32, type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size", default=8, type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate", default=5e-5, type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs", default=3.0, type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion", default=0.1, type=float,
help="Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps", default=1000, type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda", default=False, action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank", type=int, default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument("--gpu_ids", type=str, default="0",
help="select one gpu to use")
parser.add_argument('--seed', type=int, default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
help="Number of updates steps to accumualte before performing a backward/update pass.")
parser.add_argument('--optimize_on_cpu', default=False,action='store_true',
help="Whether to perform optimization and keep the optimizer averages on CPU")
parser.add_argument('--fp16', default=False, action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale', type=float, default=128,
help='Loss scaling, positive power of 2 values can improve fp16 convergence.')
################## atec
parser.add_argument('--do_submit', default=False, action='store_true', help="submit to results to atec cloud")
parser.add_argument('--train_devset', default=False, action='store_true', help="")
parser.add_argument("--test_in_file", default=None, type=str, help="")
parser.add_argument("--test_out_file", default=None, type=str, help="")
args = parser.parse_args()
if args.local_rank == -1 and not args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
#device = torch.device("cuda", args.gpu_id)
n_gpu = len(args.gpu_ids.split(','))#torch.cuda.device_count()
elif args.no_cuda:
device = torch.device('cpu')
n_gpu = 0
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info("16-bits training currently not supported in distributed training")
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
#if not args.do_train and not args.do_eval:
# raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format(
args.max_seq_length, bert_config.max_position_embeddings))
#if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir) #, exist_ok=True)
processor = AtecProcessor()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
if args.train_devset:
eval_examples = processor.get_dev_examples(args.data_dir)
train_examples += eval_examples
num_train_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
# Prepare model
logger.info('build model')
model = BertForSequenceClassification(bert_config, len(label_list))
#model = BertSiameseModel(bert_config, len(label_list))
if args.init_checkpoint is not None:
try:
# just model.bert
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
except RuntimeError as e:
# all model
import re
new_state_dict = collections.OrderedDict()
state_dict = torch.load(args.init_checkpoint, map_location='cpu')
for key in state_dict.keys():
new_key = re.sub("module\.", "", key)
new_state_dict[new_key] = state_dict[key]
model.load_state_dict(new_state_dict)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model, device_ids=[int(x) for x in
args.gpu_ids.split(',')])
global_step = 0
tr_loss=None
if args.do_train:
# Prepare optimizer
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
from tqdm import tqdm, trange
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
# train_features = convert_examples_to_siamese_features(
# train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in train_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in train_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in train_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in train_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in train_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in train_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
# train_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
with tqdm(train_dataloader, desc="Iteration") as pbar:
for step, batch in enumerate(pbar):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
# tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch
# loss, logits = model(tokens_a, types_a, mask_a, tokens_b, types_b,
# mask_b, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
preds = th.argmax(logits, dim=1)
acc = accuracy_score(label_ids.detach().cpu().numpy(),
preds.detach().cpu().numpy())
pbar.set_postfix(loss=loss.item(), acc=acc.item())
#nb_tr_examples += input_ids.size(0)
nb_tr_examples += label_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), os.path.join(args.output_dir,
"pytorch_model.bin"))
if args.do_eval:
from tqdm import tqdm, trange
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
# eval_features = convert_examples_to_siamese_features(
# eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
# eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
y_true = []
y_pred = []
for batch in tqdm(eval_dataloader):
# tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch
input_ids, input_mask, segment_ids, label_ids = batch
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
# tmp_eval_loss, logits = model(tokens_a, types_a, mask_a,
# tokens_b, types_b, mask_b, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
preds = np.argmax(logits, axis=1)
y_true.extend(label_ids)
y_pred.extend(preds)
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
#nb_eval_examples += input_ids.size(0)
nb_eval_examples += label_ids.size # np.array
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'precision': precision_score(y_true, y_pred),
'recall': recall_score(y_true, y_pred),
'f1': f1_score(y_true, y_pred)}
if tr_loss is not None:
result['loss'] = tr_loss/nb_tr_steps
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "a") as writer:
logger.info("***** Eval results *****")
writer.write("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_submit:
eval_examples = processor.get_test_examples(args.test_in_file)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
# eval_features = convert_examples_to_siamese_features(
# eval_examples, label_list, args.max_seq_length, tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
#eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long)
# all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long)
# all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long)
# all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long)
# all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long)
# all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long)
# all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
# eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a,
# all_tokens_b, all_types_b, all_mask_b, all_label_ids)
#eval_sampler = SequentialSampler(eval_data)
#eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
y_pred = []
batch_size = args.eval_batch_size
for i in range(0, len(all_label_ids), batch_size):
input_ids = all_input_ids[i:i+batch_size]
input_mask = all_input_mask[i:i+batch_size]
segment_ids = all_segment_ids[i:i+batch_size]
label_ids = all_label_ids[i:i+batch_size]
# tokens_a = all_tokens_a[i:i+batch_size]
# types_a = all_types_a[i:i+batch_size]
# mask_a = all_mask_a[i:i+batch_size]
# tokens_b = all_tokens_b[i:i+batch_size]
# types_b = all_types_b[i:i+batch_size]
# mask_b = all_mask_b[i:i+batch_size]
with torch.no_grad():
logits = model(input_ids, segment_ids, input_mask)
# logits = model(tokens_a, types_a, mask_a,
# tokens_b, types_b, mask_b)
logits = logits.detach().cpu().numpy()
preds = np.argmax(logits, axis=1)
y_pred.extend(preds)
with open(args.test_out_file, "w") as writer:
for i, y in enumerate(y_pred):
writer.write('{}\t{}\n'.format(i+1, y))
def main(args):
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
logger.info("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
args.data_dir = os.path.join(args.data_dir, args.task_name)
args.output_dir = os.path.join(args.output_dir, args.task_name)
logger.info("args = %s", args)
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
"sst-2": Sst2Processor,
"sts-b": StsbProcessor,
"qqp": QqpProcessor,
"qnli": QnliProcessor,
"rte": RteProcessor,
"wnli": WnliProcessor,
}
output_modes = {
"cola": "classification",
"mnli": "classification",
"mrpc": "classification",
"sst-2": "classification",
"sts-b": "regression",
"qqp": "classification",
"qnli": "classification",
"rte": "classification",
"wnli": "classification",
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
logger.info("Output directory already exists and is not empty.")
if not os.path.exists(args.output_dir):
try:
os.makedirs(args.output_dir)
except:
pass
logger.info("catch a error")
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.vocab_file, do_lower_case=args.do_lower_case)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(PYTORCH_PRETRAINED_BERT_CACHE,
'distributed_{}'.format(args.local_rank))
# use bert to aug train_examples
ori_train_examples = processor.get_train_examples(args.data_dir)
eval_examples = processor.get_dev_examples(args.data_dir)
if args.double_ori == 0:
num_train_optimization_steps = int(
len(ori_train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
else:
num_train_optimization_steps = int(
len(ori_train_examples) * 2 / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size()
if args.use_saved == 1:
bert_saved_dir = args.ckpt
if args.co_training:
model = BertForNSP_co.from_pretrained(bert_saved_dir,
cache_dir=args.ckpt_cache_dir,
num_labels=num_labels,
args=args)
elif args.only_bert:
model = BertForSequenceClassification.from_pretrained(bert_saved_dir,
cache_dir=args.ckpt_cache_dir,
num_labels=num_labels)
else:
model = BertForNSPAug.from_pretrained(bert_saved_dir,
cache_dir=args.ckpt_cache_dir,
num_labels=num_labels,
args=args)
else:
if args.only_bert:
model = BertForSequenceClassification.from_pretrained(args.bert_model,
cache_dir=cache_dir,
num_labels=num_labels)
else:
model = BertForNSPAug.from_pretrained(args.bert_model,
cache_dir=cache_dir,
num_labels=num_labels,
args=args)
model.cuda()
if n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.do_first_eval:
args.do_train = False
res_file = os.path.join(args.output_dir, "first_test.tsv")
eval_loss, eval_seq_loss, eval_aug_loss, eval_res, eval_aug_accuracy, res_parts = \
do_evaluate(args, processor, label_list, tokenizer, model, 0, output_mode, num_labels, task_name,
eval_examples, type="dev")
eval_res.update(res_parts)
for key in sorted(eval_res.keys()):
logger.info("first evaluation: %s = %s", key, str(eval_res[key]))
idx, preds = do_test(args, label_list, task_name, processor, tokenizer, output_mode, model)
dataframe = pd.DataFrame({'index': range(idx), 'prediction': preds})
dataframe.to_csv(res_file, index=False, sep='\t')
logger.info(" Num test length = %d", idx)
logger.info(" Done ")
# write mm test results
if task_name == "mnli":
res_file = os.path.join(args.output_dir,
"first_test_mm.tsv")
idx, preds = do_test(args, label_list, task_name, processor, tokenizer, output_mode, model, do_mm=True)
dataframe = pd.DataFrame({'index': range(idx), 'prediction': preds})
dataframe.to_csv(res_file, index=False, sep='\t')
logger.info(" Num test length = %d", idx)
logger.info(" Done write mm")
if args.do_train:
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
global_step = 0
best_val_acc = 0.0
first_time = time.time()
logger.info("***** Running training *****")
logger.info(" Num original examples = %d", len(ori_train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
model.train()
aug_ratio = 0.0
aug_seed = np.random.randint(0, 1000)
for epoch in range(int(args.num_train_epochs)):
if args.only_bert:
train_features = convert_examples_to_features(ori_train_examples, label_list, args.max_seq_length,
tokenizer, num_show=args.num_show, output_mode=output_mode, args=args)
else:
logger.info("epoch=%d, aug_ratio = %f, aug_seed=%d", epoch, aug_ratio, aug_seed)
train_examples = Aug_each_ckpt(ori_train_examples, label_list, model, tokenizer, args=args,
num_show=args.num_show, output_mode=output_mode, seed=aug_seed,
aug_ratio=aug_ratio, use_bert=False)
if aug_ratio + args.aug_ratio_each < 1.0:
aug_ratio += args.aug_ratio_each
aug_seed += 1
train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer,
num_show=args.num_show, output_mode=output_mode, args=args)
logger.info("Done convert features")
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float)
token_real_label = torch.tensor([f.token_real_label for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, token_real_label)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
logger.info("begin training")
tr_loss, tr_seq_loss, tr_aug_loss, train_seq_accuracy, train_aug_accuracy = 0, 0, 0, 0, 0
nb_tr_examples, nb_tr_steps, nb_tr_tokens = 0, 0, 0
preds = []
all_labels = []
for step, batch in enumerate(train_dataloader):
batch = tuple(t.cuda() for t in batch)
input_ids, input_mask, segment_ids, label_ids, token_real_label = batch
if args.only_bert:
seq_logits = model(input_ids, segment_ids, input_mask, labels=None)
else:
seq_logits, aug_logits, aug_loss = model(input_ids, segment_ids, input_mask, labels=None, token_real_label=token_real_label)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
seq_loss = loss_fct(seq_logits.view(-1, num_labels), label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
seq_loss = loss_fct(seq_logits.view(-1), label_ids.view(-1))
token_real_label = token_real_label.detach().cpu().numpy()
w = args.aug_loss_weight
if args.only_bert:
loss = seq_loss
else:
loss = (1 - w) * seq_loss + w * aug_loss
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 10000.0)
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
batch_loss = seq_loss.mean().item()
tr_seq_loss += seq_loss.mean().item()
seq_logits = seq_logits.detach().cpu().numpy()
label_ids = label_ids.detach().cpu().numpy()
if len(preds) == 0:
preds.append(seq_logits)
all_labels.append(label_ids)
else:
preds[0] = np.append(preds[0], seq_logits, axis=0)
all_labels[0] = np.append(all_labels[0], label_ids,axis=0)
if args.only_bert == 0:
aug_logits = aug_logits.detach().cpu().numpy()
tmp_train_aug_accuracy, tmp_tokens = accuracy(aug_logits, token_real_label, type="aug")
train_aug_accuracy += tmp_train_aug_accuracy
nb_tr_tokens += tmp_tokens
tr_aug_loss += aug_loss.mean().item()
if global_step % 20 == 0:
loss = tr_loss / nb_tr_steps
seq_loss = tr_seq_loss / nb_tr_steps
aug_loss = tr_aug_loss / nb_tr_steps
tmp_pred = preds[0]
tmp_labels = all_labels[0]
if output_mode == "classification":
tmp_pred = np.argmax(tmp_pred, axis=1)
elif output_mode == "regression":
tmp_pred = np.squeeze(tmp_pred)
res = accuracy(tmp_pred, tmp_labels, task_name=task_name)
if nb_tr_tokens != 0:
aug_avg = train_aug_accuracy / nb_tr_tokens
else:
aug_avg = 0.0
log_string = ""
log_string += "epoch={:<5d}".format(epoch)
log_string += " step={:<9d}".format(global_step)
log_string += " total_loss={:<9.7f}".format(loss)
log_string += " seq_loss={:<9.7f}".format(seq_loss)
log_string += " aug_loss={:<9.7f}".format(aug_loss)
log_string += " batch_loss={:<9.7f}".format(batch_loss)
log_string += " lr={:<9.7f}".format(optimizer.get_lr()[0])
log_string += " |g|={:<9.7f}".format(total_norm)
#log_string += " tr_seq_acc={:<9.7f}".format(seq_avg)
log_string += " tr_aug_acc={:<9.7f}".format(aug_avg)
log_string += " mins={:<9.2f}".format(float(time.time() - first_time) / 60)
for key in sorted(res.keys()):
log_string += " " + key + "= " + str(res[key])
logger.info(log_string)
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step()
optimizer.zero_grad()
global_step += 1
train_loss = tr_loss / nb_tr_steps
if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and epoch % 1 == 0:
tot_time = float(time.time() - first_time) / 60
eval_loss, eval_seq_loss, eval_aug_loss, eval_res, eval_aug_accuracy, res_parts=\
do_evaluate(args, processor, label_list, tokenizer, model, epoch, output_mode, num_labels, task_name, eval_examples, type="dev")
eval_res["tot_time"] = tot_time
if "acc" in eval_res:
tmp_acc = eval_res["acc"]
elif "mcc" in eval_res:
tmp_acc = eval_res["mcc"]
else:
tmp_acc = eval_res["corr"]
if tmp_acc >= best_val_acc:
best_val_acc = tmp_acc
dev_test = "dev"
model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self
output_model_dir = os.path.join(args.output_dir, "dev_" + str(tmp_acc))
if not os.path.exists(output_model_dir):
os.makedirs(output_model_dir)
output_model_file = os.path.join(output_model_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(output_model_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
result = {'eval_total_loss': eval_loss,
'eval_seq_loss': eval_seq_loss,
'eval_aug_loss': eval_aug_loss,
'eval_aug_accuracy': eval_aug_accuracy,
'global_step': global_step,
'train_loss': train_loss,
'best_epoch': epoch,
'train_batch_size': args.train_batch_size,
'args': args}
result.update(eval_res)
result.update(res_parts)
output_eval_file = os.path.join(args.output_dir,
dev_test + "_results_" + str(tmp_acc) + ".txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Test results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
# write test results
if args.do_test:
res_file = os.path.join(args.output_dir,
"test_" + str(tmp_acc)+".tsv")
idx, preds = do_test(args, label_list, task_name, processor, tokenizer, output_mode, model)
dataframe = pd.DataFrame({'index': range(idx), 'prediction': preds})
dataframe.to_csv(res_file, index=False, sep='\t')
logger.info(" Num test length = %d", idx)
logger.info(" Done ")
# write mm test results
if task_name == "mnli":
res_file = os.path.join(args.output_dir,
"test_mm_" + str(tmp_acc) + ".tsv")
idx, preds = do_test(args, label_list, task_name, processor, tokenizer, output_mode, model, do_mm=True)
dataframe = pd.DataFrame({'index': range(idx), 'prediction': preds})
dataframe.to_csv(res_file, index=False, sep='\t')
logger.info(" Num test length = %d", idx)
logger.info(" Done write mm")
else:
logger.info(" tmp_val_acc = %f", tmp_acc)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--saved_model",
default=None,
type=str,
help="saved model")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_predict",
action='store_true',
help="Whether to run test on the test set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=16,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--predict_batch_size",
default=8,
type=int,
help="Total batch size for predict.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=2.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=47,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cpi": BioBERTChemprotProcessor,
}
output_modes = {
"cpi": "classification",
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(
args.bert_model,
do_lower_case=args.do_lower_case,
never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "CPR:3",
"CPR:4", "CPR:5", "CPR:6", "CPR:9"))
tokenizer.vocab["CPR:3"] = 3
tokenizer.vocab["CPR:4"] = 4
tokenizer.vocab["CPR:5"] = 5
tokenizer.vocab["CPR:6"] = 6
tokenizer.vocab["CPR:9"] = 9
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
model = BertForSequenceClassification.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=num_labels)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps,
e=1e-08)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_title_ids = torch.tensor([f.title_ids for f in train_features],
dtype=torch.long)
all_title_mask = torch.tensor([f.title_mask for f in train_features],
dtype=torch.long)
all_title_segment = torch.tensor(
[f.title_segment for f in train_features], dtype=torch.long)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_sdp_ids = torch.tensor([f.sdp_ids for f in train_features],
dtype=torch.long)
all_sdp_mask = torch.tensor([f.sdp_mask for f in train_features],
dtype=torch.long)
all_sdp_segment = torch.tensor([f.sdp_segment for f in train_features],
dtype=torch.long)
all_P_gauss1_list = torch.tensor(
[f.P_gauss1_list for f in train_features], dtype=torch.float)
all_P_gauss2_list = torch.tensor(
[f.P_gauss2_list for f in train_features], dtype=torch.float)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.float)
train_data = TensorDataset(all_title_ids, all_title_mask, all_title_segment, all_input_ids, all_input_mask, all_segment_ids, \
all_sdp_ids, all_sdp_mask, all_sdp_segment, all_P_gauss1_list, all_P_gauss2_list,all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
title_ids, title_mask, title_segment, input_ids, input_mask, segment_ids, \
sdp_ids, sdp_mask, sdp_segment, P_gauss1_list, P_gauss2_list, label_ids = batch
# define a new function to compute loss values for both output_modes
logits = model(title_ids, title_segment, title_mask, input_ids, segment_ids, input_mask, \
sdp_ids, sdp_segment, sdp_mask, P_gauss1_list, P_gauss2_list, labels=None)
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), label_ids.view(-1))
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
optimizer.zero_grad()
global_step += 1
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
else:
output_model_file = os.path.join(args.saved_model, 'pytorch_model.bin')
output_config_file = os.path.join(args.saved_model, 'bert_config.json')
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config, num_labels=num_labels)
model.load_state_dict(torch.load(output_model_file))
model.to(device)
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_title_ids = torch.tensor([f.title_ids for f in eval_features],
dtype=torch.long)
all_title_mask = torch.tensor([f.title_mask for f in eval_features],
dtype=torch.long)
all_title_segment = torch.tensor(
[f.title_segment for f in eval_features], dtype=torch.long)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_sdp_ids = torch.tensor([f.sdp_ids for f in eval_features],
dtype=torch.long)
all_sdp_mask = torch.tensor([f.sdp_mask for f in eval_features],
dtype=torch.long)
all_sdp_segment = torch.tensor([f.sdp_segment for f in eval_features],
dtype=torch.long)
all_P_gauss1_list = torch.tensor(
[f.P_gauss1_list for f in eval_features], dtype=torch.float)
all_P_gauss2_list = torch.tensor(
[f.P_gauss2_list for f in eval_features], dtype=torch.float)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.float)
eval_data = TensorDataset(all_title_ids, all_title_mask, all_title_segment, all_input_ids, all_input_mask, all_segment_ids, \
all_sdp_ids, all_sdp_mask, all_sdp_segment, all_P_gauss1_list, all_P_gauss2_list, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []
for title_ids, title_mask, title_segment, input_ids, input_mask, segment_ids, sdp_ids, sdp_mask, sdp_segment, P_gauss1_list, P_gauss2_list, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
title_ids = title_ids.to(device)
title_mask = title_mask.to(device)
title_segment = title_segment.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
sdp_ids = sdp_ids.to(device)
sdp_mask = sdp_mask.to(device)
sdp_segment = sdp_segment.to(device)
P_gauss1_list = P_gauss1_list.to(device)
P_gauss2_list = P_gauss2_list.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(title_ids,
title_segment,
title_mask,
input_ids,
segment_ids,
input_mask,
sdp_ids,
sdp_segment,
sdp_mask,
P_gauss1_list,
P_gauss2_list,
labels=None)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_eval_loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = preds[0]
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
elif output_mode == "regression":
preds = np.squeeze(preds)
result = compute_metrics(task_name, preds, all_label_ids.numpy())
loss = tr_loss / nb_tr_steps if args.do_train else None
result['eval_loss'] = eval_loss
result['global_step'] = global_step
result['loss'] = loss
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if args.do_predict and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
test_examples = processor.get_test_examples(args.data_dir)
test_features = convert_examples_to_features(test_examples, label_list,
args.max_seq_length,
tokenizer, output_mode)
logger.info("***** Running prediction *****")
logger.info(" Num examples = %d", len(test_examples))
logger.info(" Batch size = %d", args.predict_batch_size)
all_title_ids = torch.tensor([f.title_ids for f in test_features],
dtype=torch.long)
all_title_mask = torch.tensor([f.title_mask for f in test_features],
dtype=torch.long)
all_title_segment = torch.tensor(
[f.title_segment for f in test_features], dtype=torch.long)
all_input_ids = torch.tensor([f.input_ids for f in test_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features],
dtype=torch.long)
all_sdp_ids = torch.tensor([f.sdp_ids for f in test_features],
dtype=torch.long)
all_sdp_mask = torch.tensor([f.sdp_mask for f in test_features],
dtype=torch.long)
all_sdp_segment = torch.tensor([f.sdp_segment for f in test_features],
dtype=torch.long)
all_P_gauss1_list = torch.tensor(
[f.P_gauss1_list for f in test_features], dtype=torch.float)
all_P_gauss2_list = torch.tensor(
[f.P_gauss2_list for f in test_features], dtype=torch.float)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in test_features],
dtype=torch.float)
predict_data = TensorDataset(all_title_ids, all_title_mask,
all_title_segment, all_input_ids,
all_input_mask, all_segment_ids,
all_sdp_ids, all_sdp_mask,
all_sdp_segment, all_P_gauss1_list,
all_P_gauss2_list, all_label_ids)
# Run prediction for full data
predict_sampler = SequentialSampler(predict_data)
predict_dataloader = DataLoader(predict_data,
sampler=predict_sampler,
batch_size=args.predict_batch_size)
model.eval()
test_loss = 0
nb_test_steps = 0
preds = []
for title_ids, title_mask, title_segment, input_ids, input_mask, segment_ids, sdp_ids, sdp_mask, sdp_segment, P_gauss1_list, P_gauss2_list, label_ids in tqdm(
predict_dataloader, desc="Predicting"):
title_ids = title_ids.to(device)
title_mask = title_mask.to(device)
title_segment = title_segment.to(device)
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
sdp_ids = sdp_ids.to(device)
sdp_mask = sdp_mask.to(device)
sdp_segment = sdp_segment.to(device)
P_gauss1_list = P_gauss1_list.to(device)
P_gauss2_list = P_gauss2_list.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits = model(title_ids, title_segment, title_mask, input_ids, segment_ids, input_mask,\
sdp_ids, sdp_segment, sdp_mask, P_gauss1_list, P_gauss2_list, labels=None)
# create eval loss and other metric required by the task
if output_mode == "classification":
loss_fct = CrossEntropyLoss()
tmp_test_loss = loss_fct(logits.view(-1, num_labels),
label_ids.view(-1))
elif output_mode == "regression":
loss_fct = MSELoss()
tmp_test_loss = loss_fct(logits.view(-1), label_ids.view(-1))
test_loss += tmp_test_loss.mean().item()
nb_test_steps += 1
if len(preds) == 0:
preds.append(logits.detach().cpu().numpy())
else:
preds[0] = np.append(preds[0],
logits.detach().cpu().numpy(),
axis=0)
test_loss = test_loss / nb_test_steps
preds = preds[0]
if output_mode == "classification":
preds = np.argmax(preds, axis=1)
elif output_mode == "regression":
preds = np.squeeze(preds)
result = compute_metrics(task_name, preds, all_label_ids.numpy())
loss = tr_loss / nb_tr_steps if args.do_train else None
result['test_loss'] = test_loss
result['global_step'] = global_step
result['loss'] = loss
output_test_file = os.path.join(args.output_dir, "test_results.txt")
with open(output_test_file, "w") as writer:
logger.info("***** Predict results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def train(args):
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
'sst': SstProcessor,
'polarity': PolarityProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError("Invalid accumulate_gradients parameter: {}, should be >= 1".format(
args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size / args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval and not args.do_test:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format(
args.max_seq_length, bert_config.max_position_embeddings))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs)
model = BertForSequenceClassification(bert_config, len(label_list))
if args.do_test:
model.load_state_dict(torch.load(os.path.join(args.data_dir, "model_best.pt")))
model.to(device)
model.eval()
if args.do_eval:
eval_example = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_example, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running eval *****")
logger.info(" Num examples = %d", len(eval_example))
logger.info(" Batch size = %d", args.eval_batch_size)
_ = do_eval(model, device, eval_features, args)
if args.do_predict:
print(os.path.dirname(args.data_dir))
test_example = processor.get_test_examples(os.path.dirname(args.data_dir))
test_features = convert_examples_to_features(
test_example, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running test *****")
logger.info(" Num examples = %d", len(test_example))
logger.info(" Batch size = %d", args.eval_batch_size)
test_logit_list, _ = do_eval(model, device, test_features, args)
np.save(os.path.join(args.data_dir, 'oof_test'), np.asarray(test_logit_list))
return 0
if args.init_checkpoint is not None:
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [
{'params': [p for n, p in model.named_parameters() if n not in no_decay], 'weight_decay_rate': 0.01},
{'params': [p for n, p in model.named_parameters() if n in no_decay], 'weight_decay_rate': 0.0}
]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
max_score = 0
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, (input_ids, input_mask, segment_ids, label_ids) in enumerate(
tqdm(train_dataloader, desc="Iteration")):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
logit_list = []
labels_eval_list = []
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
logit_list.extend(logits.tolist())
labels_eval_list.extend(label_ids.tolist())
tmp_eval_accuracy = accuracy(logits, label_ids)
# _ = accuracy2(logits, label_ids)
# _ = accuracy3(logits, label_ids)
# _ = accuracy4(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
# print(epoch)
# f1 = score2(logit_list, labels_eval_list)
eval_loss = eval_loss / nb_eval_steps # len(eval_dataloader)
eval_accuracy = eval_accuracy / nb_eval_examples # len(eval_dataloader)
print("eval_loss", eval_loss)
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps,
'Epoch': epoch,
'Dir': args.data_dir
} # 'loss': loss.item()}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "a") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
if eval_accuracy > max_score:
# best_logit_list = logit_list
# best_eval_labels = labels_eval_list
np.save(os.path.join(args.data_dir, 'oof_train'), np.asarray(logit_list))
np.save(os.path.join(args.data_dir, 'oof_train_y'), np.asarray(labels_eval_list))
torch.save(model.state_dict(), os.path.join(args.data_dir, "model_%.4f.pt" % eval_accuracy))
if os.path.exists(os.path.join(args.data_dir, "model_best.pt")):
os.remove(os.path.join(args.data_dir, "model_best.pt"))
torch.save(model.state_dict(), os.path.join(args.data_dir, "model_best.pt"))
max_score = eval_accuracy
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_config_file",
default=None,
type=str,
required=True,
help=
"The config json file corresponding to the pre-trained BERT model. \n"
"This specifies the model architecture.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--vocab_file",
default=None,
type=str,
required=True,
help="The vocabulary file that the BERT model was trained on.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--init_checkpoint",
default=None,
type=str,
help="Initial checkpoint (usually from a pre-trained BERT model).")
parser.add_argument(
"--do_lower_case",
default=False,
action='store_true',
help=
"Whether to lower case the input text. True for uncased models, False for cased models."
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
default=False,
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
default=False,
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--save_checkpoints_steps",
default=1000,
type=int,
help="How often to save the model checkpoint.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument(
"--accumulate_gradients",
type=int,
default=1,
help=
"Number of steps to accumulate gradient on (divide the batch_size and accumulate)"
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass."
)
args = parser.parse_args()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info("device %s n_gpu %d distributed training %r", device, n_gpu,
bool(args.local_rank != -1))
if args.accumulate_gradients < 1:
raise ValueError(
"Invalid accumulate_gradients parameter: {}, should be >= 1".
format(args.accumulate_gradients))
args.train_batch_size = int(args.train_batch_size /
args.accumulate_gradients)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = BertConfig.from_json_file(args.bert_config_file)
if args.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}"
.format(args.max_seq_length, bert_config.max_position_embeddings))
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty.".format(
args.output_dir))
os.makedirs(args.output_dir, exist_ok=True)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int(
len(train_examples) / args.train_batch_size *
args.num_train_epochs)
model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None:
model.bert.load_state_dict(
torch.load(args.init_checkpoint, map_location='cpu'))
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
no_decay = ['bias', 'gamma', 'beta']
optimizer_parameters = [{
'params':
[p for n, p in model.named_parameters() if n not in no_decay],
'weight_decay_rate':
0.01
}, {
'params': [p for n, p in model.named_parameters() if n in no_decay],
'weight_decay_rate':
0.0
}]
optimizer = BERTAdam(optimizer_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() # We have accumulated enought gradients
model.zero_grad()
global_step += 1
if args.do_eval:
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, segment_ids,
input_mask, label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {
'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': tr_loss / nb_tr_steps
}
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default=None,
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
parser.add_argument("--init_checkpoint",
default=None,
type=str,
required=True,
help="The checkpoint file from pretraining")
## Other parameters
parser.add_argument(
"--cache_dir",
default="",
type=str,
help=
"Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--google_pretrained",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--max_steps",
default=-1.0,
type=float,
help="Total number of training steps to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=1,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument('--server_ip',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="Can be used for distant debugging.")
parser.add_argument("--old",
action='store_true',
help="use old fp16 optimizer")
parser.add_argument(
'--vocab_file',
type=str,
default=None,
required=True,
help="Vocabulary mapping/file BERT was pretrainined on")
parser.add_argument("--config_file",
default=None,
type=str,
required=True,
help="The BERT model config")
args = parser.parse_args()
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
processors = {
"cola": ColaProcessor,
"mnli": MnliProcessor,
"mrpc": MrpcProcessor,
}
num_labels_task = {
"cola": 2,
"mnli": 3,
"mrpc": 2,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train:
print(
"WARNING: Output directory ({}) already exists and is not empty.".
format(args.output_dir))
if not os.path.exists(args.output_dir) and is_main_process():
os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
#tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
tokenizer = BertTokenizer(args.vocab_file,
do_lower_case=args.do_lower_case,
max_len=512) # for bert large
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size /
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare model
config = BertConfig.from_json_file(args.config_file)
# Padding for divisibility by 8
if config.vocab_size % 8 != 0:
config.vocab_size += 8 - (config.vocab_size % 8)
model = BertForSequenceClassification(config, num_labels=num_labels)
print("USING CHECKPOINT from", args.init_checkpoint)
model.load_state_dict(torch.load(args.init_checkpoint,
map_location='cpu')["model"],
strict=False)
print("USED CHECKPOINT from", args.init_checkpoint)
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
if args.fp16:
print("using fp16")
try:
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False)
if args.loss_scale == 0:
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O2",
keep_batchnorm_fp32=False,
loss_scale="dynamic")
else:
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O2",
keep_batchnorm_fp32=False,
loss_scale=args.loss_scale)
scheduler = LinearWarmUpScheduler(
optimizer,
warmup=args.warmup_proportion,
total_steps=num_train_optimization_steps)
else:
print("using fp32")
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
print("data prep")
cached_train_features_file = args.data_dir + '_{0}_{1}_{2}'.format(
list(filter(None, args.bert_model.split('/'))).pop(),
str(args.max_seq_length), str(args.do_lower_case))
train_features = None
try:
with open(cached_train_features_file, "rb") as reader:
train_features = pickle.load(reader)
except:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
if args.local_rank == -1 or torch.distributed.get_rank() == 0:
logger.info(" Saving train features into cached file %s",
cached_train_features_file)
with open(cached_train_features_file, "wb") as writer:
pickle.dump(train_features, writer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
if args.max_steps > 0 and global_step > args.max_steps:
break
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up for BERT which FusedAdam doesn't do
scheduler.step()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if args.do_eval and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features],
dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
preds = None
out_label_ids = None
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = label_ids.detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
out_label_ids = np.append(out_label_ids,
label_ids.detach().cpu().numpy(),
axis=0)
eval_loss = eval_loss / nb_eval_steps
preds = np.argmax(preds, axis=1)
eval_loss = eval_loss / nb_eval_steps
loss = tr_loss / nb_tr_steps if args.do_train else None
results = {
'eval_loss': eval_loss,
'global_step': global_step,
'loss': loss
}
result = compute_metrics(task_name, preds, out_label_ids)
results.update(result)
print(results)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(results.keys()):
logger.info(" %s = %s", key, str(results[key]))
writer.write("%s = %s\n" % (key, str(results[key])))
