def main():
parser = argparse.ArgumentParser()
parser.add_argument("--task_name",default='ner',type=str)
parser.add_argument("--do_test",action='store_true')
parser.add_argument("--do_eval",action='store_true')
parser.add_argument('--seed',default=42,type=str)
args = parser.parse_args()
seed_everything(seed=args.seed)
dt = str(datetime.today()).split(" ")[0]
test_path = config['data_dir'] / 'test.txt'
test_result_path = config['result'] / f'{dt}_submit_test.txt'
processors = {"ner": NerProcessor}
task_name = args.task_name.lower()
processor = processors[task_name]()
label_list = processor.get_labels()
id2label = {i: label for i, label in enumerate(label_list, 0)}
test_data = []
with open(str(test_path), 'r') as fr:
for line in fr:
line = line.strip("\n")
test_data.append(line)
fw = open(str(test_result_path), 'w')
cv_test_pred = []
for file in glob(f"{str(config['result']/ '*.pkl')}"):
data = load_pickle(file)
cv_test_pred.append(data)
vote_pred = []
for i in range(len(test_data)):
t = [np.array([x[i]]).T for x in cv_test_pred]
t2 = np.concatenate(t, axis=1)
t3 = []
for line in t2:
c = Counter()
c.update(line)
t3.append(c.most_common(1)[0][0])
vote_pred.append(t3)
for tag,line in zip(vote_pred,test_data):
token_a = line.split("_")
label_entities = get_entities(tag, id2label)
if len(label_entities) == 0:
record = "_".join(token_a) + "/o"
else:
labels = []
label_entities = sorted(label_entities, key=lambda x: x[1])
o_s = 0
for i, entity in enumerate(label_entities):
begin = entity[1]
end = entity[2]
tag = entity[0]
if begin != o_s:
labels.append("_".join(token_a[o_s:begin]) + "/o")
labels.append("_".join(token_a[begin:end + 1]) + f"/{tag}")
o_s = end + 1
if i == len(label_entities) - 1:
if o_s <= len(token_a) - 1:
labels.append("_".join(token_a[o_s:]) + "/o")
record = " ".join(labels)
fw.write(record + "\n")
fw.close()
def main():
parser = ArgumentParser()
parser.add_argument("--arch", default='bert_lstm_span', type=str)
parser.add_argument("--do_train", action='store_true')
parser.add_argument("--do_test", action='store_true')
parser.add_argument("--save_best", action='store_true')
parser.add_argument("--do_lower_case", action='store_true')
parser.add_argument('--soft_label', action='store_true')
parser.add_argument('--data_name', default='datagrand', type=str)
parser.add_argument('--optimizer',
default='adam',
type=str,
choices=['adam', 'lookahead'])
parser.add_argument('--markup',
default='bios',
type=str,
choices=['bio', 'bios'])
parser.add_argument('--checkpoint', default=900000, type=int)
parser.add_argument('--fold', default=0, type=int)
parser.add_argument("--epochs", default=50.0, type=int)
parser.add_argument("--resume_path", default='', type=str)
parser.add_argument("--mode", default='max', type=str)
parser.add_argument("--monitor", default='valid_f1', type=str)
parser.add_argument("--local_rank", type=int, default=-1)
parser.add_argument("--sorted",
default=1,
type=int,
help='1 : True 0:False ')
parser.add_argument("--n_gpu",
type=str,
default='0',
help='"0,1,.." or "0" or "" ')
parser.add_argument('--gradient_accumulation_steps', type=int, default=1)
parser.add_argument("--train_batch_size", default=24, type=int)
parser.add_argument('--eval_batch_size', default=48, type=int)
parser.add_argument("--train_max_seq_len", default=128, type=int)
parser.add_argument("--eval_max_seq_len", default=512, type=int)
parser.add_argument('--loss_scale', type=float, default=0)
parser.add_argument("--warmup_proportion", default=0.1, type=float)
parser.add_argument("--weight_decay", default=0.01, type=float)
parser.add_argument("--adam_epsilon", default=1e-8, type=float)
parser.add_argument("--grad_clip", default=5.0, type=float)
parser.add_argument("--learning_rate", default=1e-4, type=float)
parser.add_argument('--seed', type=int, default=42)
parser.add_argument("--no_cuda", action='store_true')
parser.add_argument('--fp16', action='store_true')
parser.add_argument('--fp16_opt_level', type=str, default='O1')
args = parser.parse_args()
args.pretrain_model = config[
'checkpoint_dir'] / f'lm-checkpoint-{args.checkpoint}'
args.device = torch.device(
f"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.arch = args.arch + f"_{args.markup}_fold_{args.fold}"
if args.optimizer == 'lookahead':
args.arch += "_lah"
args.model_path = config['checkpoint_dir'] / args.arch
args.model_path.mkdir(exist_ok=True)
# Good practice: save your training arguments together with the trained model
torch.save(args, config['checkpoint_dir'] / 'training_args.bin')
seed_everything(args.seed)
init_logger(log_file=config['log_dir'] / f"{args.arch}.log")
logger.info("Training/evaluation parameters %s", args)
if args.do_train:
run_train(args)
if args.do_test:
run_test(args)
def main():
parser = ArgumentParser()
parser.add_argument("--do_data", default=False, action='store_true')
parser.add_argument("--do_corpus", default=False, action='store_true')
parser.add_argument("--do_vocab", default=False, action='store_true')
parser.add_argument("--do_split", default=False, action='store_true')
parser.add_argument('--seed', default=42, type=int)
parser.add_argument('--min_freq', default=0, type=int)
parser.add_argument("--line_per_file", default=1000000000, type=int)
parser.add_argument("--file_num",
type=int,
default=10,
help="Number of dynamic masking to pregenerate")
parser.add_argument("--max_seq_len", type=int, default=128)
parser.add_argument(
"--short_seq_prob",
type=float,
default=0.1,
help="Probability of making a short sentence as a training example")
parser.add_argument(
"--masked_lm_prob",
type=float,
default=0.15,
help="Probability of masking each token for the LM task")
parser.add_argument(
"--max_predictions_per_seq",
type=int,
default=20,
help="Maximum number of tokens to mask in each sequence")
args = parser.parse_args()
seed_everything(args.seed)
vocab = Vocabulary(min_freq=args.min_freq, add_unused=False)
if args.do_corpus:
corpus = []
train_path = str(config['data_dir'] / 'train.txt')
with open(train_path, 'r') as fr:
for ex_id, line in enumerate(fr):
line = line.strip("\n")
lines = [
" ".join(x.split("/")[0].split("_"))
for x in line.split(" ")
]
if ex_id == 0:
logger.info(f"Train example: {' '.join(lines)}")
corpus.append(" ".join(lines))
test_path = str(config['data_dir'] / 'test.txt')
with open(test_path, 'r') as fr:
for ex_id, line in enumerate(fr):
line = line.strip("\n")
lines = line.split("_")
if ex_id == 0:
logger.info(f"Test example: {' '.join(lines)}")
corpus.append(" ".join(lines))
corpus_path = str(config['data_dir'] / 'corpus.txt')
with open(corpus_path, 'r') as fr:
for ex_id, line in enumerate(fr):
line = line.strip("\n")
lines = line.split("_")
if ex_id == 0:
logger.info(f"Corpus example: {' '.join(lines)}")
corpus.append(" ".join(lines))
corpus = list(set(corpus))
logger.info(f"corpus size: {len(corpus)}")
random_order = list(range(len(corpus)))
np.random.shuffle(random_order)
corpus = [corpus[i] for i in random_order]
new_corpus_path = config['data_dir'] / "corpus/corpus.txt"
if not new_corpus_path.exists():
new_corpus_path.parent.mkdir(exist_ok=True)
with open(new_corpus_path, 'w') as fr:
for line in corpus:
fr.write(line + "\n")
if args.do_split:
new_corpus_path = config['data_dir'] / "corpus/corpus.txt"
split_save_path = config['data_dir'] / "corpus/train"
if not split_save_path.exists():
split_save_path.mkdir(exist_ok=True)
line_per_file = args.line_per_file
command = f'split -a 4 -l {line_per_file} -d {new_corpus_path} {split_save_path}/shard_'
os.system(f"{command}")
if args.do_vocab:
vocab.read_data(data_path=config['data_dir'] / "corpus/train")
vocab.build_vocab()
vocab.save(file_path=config['data_dir'] / 'corpus/vocab_mapping.pkl')
vocab.save_bert_vocab(file_path=config['checkpoint_dir'] / 'vocab.txt')
logger.info(f"vocab size: {len(vocab)}")
bert_base_config['vocab_size'] = len(vocab)
save_json(data=bert_base_config,
file_path=config['checkpoint_dir'] / 'config.json')
if args.do_data:
vocab_list = vocab.load_bert_vocab(config['checkpoint_dir'] /
'vocab.txt')
data_path = config['data_dir'] / "corpus/train"
files = sorted([
f for f in data_path.iterdir() if f.exists() and "." not in str(f)
])
logger.info("--- pregenerate training data parameters ---")
logger.info(f'max_seq_len: {args.max_seq_len}')
logger.info(f"max_predictions_per_seq: {args.max_predictions_per_seq}")
logger.info(f"masked_lm_prob: {args.masked_lm_prob}")
logger.info(f"seed: {args.seed}")
logger.info(f"file num : {args.file_num}")
for idx in range(args.file_num):
logger.info(f"pregenetate file_{idx}.json")
save_filename = data_path / f"file_{idx}.json"
num_instances = 0
with save_filename.open('w') as fw:
for file_idx in range(len(files)):
file_path = files[file_idx]
file_examples = build_examples(
file_path,
max_seq_len=args.max_seq_len,
masked_lm_prob=args.masked_lm_prob,
max_predictions_per_seq=args.max_predictions_per_seq,
vocab_list=vocab_list)
file_examples = [
json.dumps(instance) for instance in file_examples
]
for instance in file_examples:
fw.write(instance + '\n')
num_instances += 1
metrics_file = data_path / f"file_{idx}_metrics.json"
print(f"num_instances: {num_instances}")
with metrics_file.open('w') as metrics_file:
metrics = {
"num_training_examples": num_instances,
"max_seq_len": args.max_seq_len
}
metrics_file.write(json.dumps(metrics))
def main():
parser = ArgumentParser()
parser.add_argument("--file_num", type=int, default=10,
help="Number of pregenerate file")
parser.add_argument("--reduce_memory", action="store_true",
help="Store training data as on-disc memmaps to massively reduce memory usage")
parser.add_argument("--epochs", type=int, default=4,
help="Number of epochs to train for")
parser.add_argument('--num_eval_steps', default=2000)
parser.add_argument('--num_save_steps', default=5000)
parser.add_argument("--local_rank", type=int, default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument("--no_cuda", action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument("--train_batch_size", default=18, type=int,
help="Total batch size for training.")
parser.add_argument('--loss_scale', type=float, default=0,
help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument("--warmup_proportion", default=0.1, type=float,
help="Linear warmup over warmup_steps.")
parser.add_argument("--adam_epsilon", default=1e-8, type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument('--max_grad_norm', default=1.0, type=float)
parser.add_argument("--learning_rate", default=2e-4, type=float,
help="The initial learning rate for Adam.")
parser.add_argument('--seed', type=int, default=42,
help="random seed for initialization")
parser.add_argument('--fp16_opt_level', type=str, default='O2',
help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument('--fp16', action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
args = parser.parse_args()
pregenerated_data = config['data_dir'] / "corpus/train"
assert pregenerated_data.is_dir(), \
"--pregenerated_data should point to the folder of files made by prepare_lm_data_mask.py!"
samples_per_epoch = 0
for i in range(args.file_num):
data_file = pregenerated_data / f"file_{i}.json"
metrics_file = pregenerated_data / f"file_{i}_metrics.json"
if data_file.is_file() and metrics_file.is_file():
metrics = json.loads(metrics_file.read_text())
samples_per_epoch += metrics['num_training_examples']
else:
if i == 0:
exit("No training data was found!")
print(f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs}).")
print("This script will loop over the available data, but training diversity may be negatively impacted.")
break
logger.info(f"samples_per_epoch: {samples_per_epoch}")
if args.local_rank == -1 or args.no_cuda:
device = torch.device(f"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
args.n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
f"device: {device} , distributed training: {bool(args.local_rank != -1)}, 16-bits training: {args.fp16}")
if args.gradient_accumulation_steps < 1:
raise ValueError(
f"Invalid gradient_accumulation_steps parameter: {args.gradient_accumulation_steps}, should be >= 1")
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
seed_everything(args.seed)
tokenizer = BertTokenizer(vocab_file=config['checkpoint_dir'] / 'vocab.txt')
total_train_examples = samples_per_epoch * args.epochs
num_train_optimization_steps = int(
total_train_examples / args.train_batch_size / args.gradient_accumulation_steps)
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size()
args.warmup_steps = int(num_train_optimization_steps * args.warmup_proportion)
# Prepare model
with open(str(config['checkpoint_dir'] / 'config.json'), "r", encoding='utf-8') as reader:
json_config = json.loads(reader.read())
print(json_config)
bert_config = BertConfig.from_json_file(str(config['checkpoint_dir'] / 'config.json'))
model = BertForMaskedLM(config=bert_config)
# model = BertForMaskedLM.from_pretrained(config['checkpoint_dir'] / 'checkpoint-580000')
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
lr_scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=num_train_optimization_steps)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
global_step = 0
metric = LMAccuracy()
tr_acc = AverageMeter()
tr_loss = AverageMeter()
train_logs = {}
logger.info("***** Running training *****")
logger.info(f" Num examples = {total_train_examples}")
logger.info(f" Batch size = {args.train_batch_size}")
logger.info(f" Num steps = {num_train_optimization_steps}")
logger.info(f" warmup_steps = {args.warmup_steps}")
seed_everything(args.seed) # Added here for reproducibility
for epoch in range(args.epochs):
for idx in range(args.file_num):
epoch_dataset = PregeneratedDataset(file_id=idx, training_path=pregenerated_data, tokenizer=tokenizer,
reduce_memory=args.reduce_memory)
if args.local_rank == -1:
train_sampler = RandomSampler(epoch_dataset)
else:
train_sampler = DistributedSampler(epoch_dataset)
train_dataloader = DataLoader(epoch_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, lm_label_ids = batch
outputs = model(input_ids=input_ids, token_type_ids=segment_ids,
attention_mask=input_mask, masked_lm_labels=lm_label_ids)
pred_output = outputs[1]
loss = outputs[0]
metric(logits=pred_output.view(-1, bert_config.vocab_size), target=lm_label_ids.view(-1))
if args.n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
nb_tr_steps += 1
tr_acc.update(metric.value(), n=input_ids.size(0))
tr_loss.update(loss.item(), n=1)
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
else:
torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
lr_scheduler.step()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if global_step % args.num_eval_steps == 0:
train_logs['loss'] = tr_loss.avg
train_logs['acc'] = tr_acc.avg
show_info = f'\n[Training]:[{epoch}/{args.epochs}]{global_step}/{num_train_optimization_steps} ' + "-".join(
[f' {key}: {value:.4f} ' for key, value in train_logs.items()])
logger.info(show_info)
tr_acc.reset()
tr_loss.reset()
if global_step % args.num_save_steps == 0:
if args.local_rank in [-1, 0] and args.num_save_steps > 0:
# Save model checkpoint
output_dir = config['checkpoint_dir'] / f'lm-checkpoint-{global_step}'
if not output_dir.exists():
output_dir.mkdir()
# save model
model_to_save = model.module if hasattr(model,
'module') else model # Take care of distributed/parallel training
model_to_save.save_pretrained(str(output_dir))
torch.save(args, str(output_dir / 'training_args.bin'))
logger.info("Saving model checkpoint to %s", output_dir)
# save config
output_config_file = output_dir / CONFIG_NAME
with open(str(output_config_file), 'w') as f:
f.write(model_to_save.config.to_json_string())
# save vocab
tokenizer.save_vocabulary(output_dir)
