def read_data_and_create_examples(self, example_type, cached_examples_file,
input_file):
if cached_examples_file.exists():
logger.info("Loading examples from cached file %s",
cached_examples_file)
examples = torch.load(cached_examples_file)
else:
# create examples
df_dataset = pd.read_csv(input_file).fillna("")
pbar = ProgressBar(n_total=len(df_dataset), desc='create examples')
examples = []
for i, row in df_dataset.iterrows():
guid = '%s-%d' % (example_type, i)
seq_id = row["id"]
text_a = row["title"]
text_b = row["content"]
label = row["label"]
label = int(label)
example = InputExample(guid=guid,
seq_id=seq_id,
text_a=text_a,
text_b=text_b,
label=label)
examples.append(example)
pbar(step=i)
logger.info("Saving examples into cached file %s",
cached_examples_file)
torch.save(examples, cached_examples_file)
return examples
def create_examples(self, lines, example_type, cached_examples_file):
'''
Creates examples for data
'''
pbar = ProgressBar(n_total=len(lines), desc='create examples')
if cached_examples_file.exists():
logger.info("Loading examples from cached file %s",
cached_examples_file)
examples = torch.load(cached_examples_file)
else:
examples = []
for i, line in enumerate(lines):
guid = '%s-%d' % (example_type, i)
text_a = line[0]
text_b = line[1]
label = line[2]
label = int(label)
example = InputExample(guid=guid,
text_a=text_a,
text_b=text_b,
label=label)
examples.append(example)
pbar(step=i)
logger.info("Saving examples into cached file %s",
cached_examples_file)
torch.save(examples, cached_examples_file)
return examples
def evaluate(args, model, eval_dataloader, metrics):
# Eval!
logger.info(" Num examples = %d", len(eval_dataloader))
logger.info(" Batch size = %d", args.eval_batch_size)
eval_loss = AverageMeter()
metrics.reset()
preds = []
targets = []
pbar = ProgressBar(n_total=len(eval_dataloader), desc='Evaluating')
for bid, batch in enumerate(eval_dataloader):
model.eval()
batch = tuple(t.to(args.device) for t in batch)
with torch.no_grad():
inputs = {
'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[3]
}
inputs['token_type_ids'] = batch[2]
outputs = model(**inputs)
loss, logits = outputs[:2]
eval_loss.update(loss.item(), n=batch[0].size()[0])
preds.append(logits.cpu().detach())
targets.append(inputs['labels'].cpu().detach())
pbar(bid)
preds = torch.cat(preds, dim=0).cpu().detach()
targets = torch.cat(targets, dim=0).cpu().detach()
metrics(preds, targets)
eval_log = {"eval_acc": metrics.value(), 'eval_loss': eval_loss.avg}
return eval_log
def is_any_photo_shared(folder):
"Checks the files in the folder to decide the image shared before or not"
logger.info("Checking for is any photo shared before")
for filename in os.listdir(folder):
logger.debug("Filename is: %s", filename)
if is_shared(os.path.join(folder, filename)):
return True
return False
def is_shared(filepath):
"Checks the file to decide the image shared before or not"
for folder in os.listdir(SHARED):
folder = os.path.join(SHARED, folder)
logger.info("folders are: %s", folder)
for filename in os.listdir(folder):
filename = os.path.join(folder, filename)
if is_similar(filepath, filename):
return True
return False
def predict(args, model, pred_dataloader, config):
# Predict (without compute metrics)
# args.predict_save_path = config['pred_dir'] / f'{args.pred_dir_name}'
# args.predict_save_path.mkdir(exist_ok=True)
logger.info(" Num examples = %d", len(pred_dataloader))
logger.info(" Batch size = %d", args.eval_batch_size)
seq_ids = []
preds = []
pbar = ProgressBar(n_total=len(pred_dataloader), desc='Predicting')
for bid, batch in enumerate(pred_dataloader):
model.eval()
batch = tuple(
t.to(args.device) if isinstance(t, torch.Tensor) else t
for t in batch)
seq_ids += list(batch[-1])
with torch.no_grad():
inputs = {
'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[3]
}
inputs['token_type_ids'] = batch[2]
##############
# writer = SummaryWriter(config["output_dir"])
# ips = {k: v[[0], ...] for k, v in inputs.items()}
# ops = model(**ips)
# model_graph_inputs = (
# ips["input_ids"], ips["attention_mask"], ips["token_type_ids"], [1,2], [3,4], ips["labels"])
# writer.add_graph(model, model_graph_inputs)
# writer.close()
##############
outputs = model(**inputs)
loss, logits = outputs[:2]
preds.append(logits.cpu().detach())
pbar(bid)
preds = torch.cat(preds, dim=0).cpu().detach()
preds_label = torch.argmax(preds, dim=1)
result_label = DataFrame(data={
"id": Series(seq_ids),
"label": Series(preds_label)
})
result_label.to_csv(config["predict_result"], index=False)
preds_softmax = torch.softmax(preds, dim=1)
result_softmax = DataFrame(
data={
"id": Series(seq_ids),
"label_0": Series(preds_softmax[:, 0]),
"label_1": Series(preds_softmax[:, 1]),
"label_2": Series(preds_softmax[:, 2])
})
result_softmax.to_csv(config["predict_softmax"], index=False)
return result_label
def main():
parser = argparse.ArgumentParser()
# parser.add_argument("--arch", default='albert_xlarge', type=str)
parser.add_argument("--arch", default='albert_large', type=str)
parser.add_argument('--bert_dir',
default='pretrain/pytorch/albert_large_zh',
type=str)
parser.add_argument('--albert_config_path',
default='configs/albert_config_large.json',
type=str)
parser.add_argument('--task_name', default='lcqmc', type=str)
parser.add_argument(
"--train_max_seq_len",
default=64,
type=int,
help=
"The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.")
parser.add_argument(
"--eval_max_seq_len",
default=64,
type=int,
help=
"The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.")
parser.add_argument('--share_type',
default='all',
type=str,
choices=['all', 'attention', 'ffn', 'None'])
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
action='store_true',
help="Whether to run eval on the test set.")
parser.add_argument(
"--evaluate_during_training",
action='store_true',
help="Rul evaluation during training at each logging step.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--eval_batch_size",
default=16,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument("--learning_rate",
default=2e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.1,
type=float,
help="Weight deay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=5.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=int,
help=
"Proportion of training to perform linear learning rate warmup for,E.g., 0.1 = 10% of training."
)
parser.add_argument(
"--eval_all_checkpoints",
action='store_true',
help=
"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number"
)
parser.add_argument("--no_cuda",
action='store_true',
help="Avoid using CUDA when available")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument(
'--overwrite_cache',
action='store_true',
help="Overwrite the cached training and evaluation sets")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--fp16',
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"
)
parser.add_argument(
'--fp16_opt_level',
type=str,
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument('--server_ip',
type=str,
default='',
help="For distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="For distant debugging.")
args = parser.parse_args()
# Fix bug: Config is wrong from base.py if it is not base
config['bert_dir'] = args.bert_dir
config['albert_config_path'] = args.albert_config_path
args.model_save_path = config['checkpoint_dir'] / f'{args.arch}'
args.model_save_path.mkdir(exist_ok=True)
# Setudistant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
args.device = device
init_logger(log_file=config['log_dir'] / 'finetuning.log')
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank, device, args.n_gpu, bool(args.local_rank != -1),
args.fp16)
# Set seed
seed_everything(args.seed)
# --------- data
processor = AlbertProcessor(vocab_path=config['albert_vocab_path'],
do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
num_labels = len(label_list)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
bert_config = AlbertConfig.from_pretrained(str(
config['albert_config_path']),
share_type=args.share_type,
num_labels=num_labels)
logger.info("Training/evaluation parameters %s", args)
metrics = Accuracy(topK=1)
# Training
if args.do_train:
train_data = processor.get_train(config['data_dir'] / "train.txt")
train_examples = processor.create_examples(
lines=train_data,
example_type='train',
cached_examples_file=config['data_dir'] /
f"cached_train_examples_{args.arch}")
train_features = processor.create_features(
examples=train_examples,
max_seq_len=args.train_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_train_features_{}_{}".format(args.train_max_seq_len,
args.arch))
train_dataset = processor.create_dataset(train_features)
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
valid_data = processor.get_dev(config['data_dir'] / "dev.txt")
valid_examples = processor.create_examples(
lines=valid_data,
example_type='valid',
cached_examples_file=config['data_dir'] /
f"cached_valid_examples_{args.arch}")
valid_features = processor.create_features(
examples=valid_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_valid_features_{}_{}".format(args.eval_max_seq_len,
args.arch))
valid_dataset = processor.create_dataset(valid_features)
valid_sampler = SequentialSampler(valid_dataset)
valid_dataloader = DataLoader(valid_dataset,
sampler=valid_sampler,
batch_size=args.eval_batch_size)
model = AlbertForSequenceClassification.from_pretrained(
config['bert_dir'], config=bert_config)
if args.local_rank == 0:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
model.to(args.device)
train(args, train_dataloader, valid_dataloader, metrics, model)
if args.do_test:
test_data = processor.get_train(config['data_dir'] / "test.txt")
test_examples = processor.create_examples(
lines=test_data,
example_type='test',
cached_examples_file=config['data_dir'] /
f"cached_test_examples_{args.arch}")
test_features = processor.create_features(
examples=test_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_test_features_{}_{}".format(args.eval_max_seq_len,
args.arch))
test_dataset = processor.create_dataset(test_features)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset,
sampler=test_sampler,
batch_size=args.eval_batch_size)
model = AlbertForSequenceClassification.from_pretrained(
args.model_save_path, config=bert_config)
model.to(args.device)
test_log = evaluate(args, model, test_dataloader, metrics)
print(test_log)
def train(args, train_dataloader, eval_dataloader, metrics, model):
""" Train the model """
t_total = len(train_dataloader
) // args.gradient_accumulation_steps * args.num_train_epochs
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay':
args.weight_decay
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay':
0.0
}]
args.warmup_steps = t_total * args.warmup_proportion
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=args.warmup_steps,
t_total=t_total)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
# multi-gpu training (should be after apex fp16 initialization)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Distributed training (should be after apex fp16 initialization)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank,
find_unused_parameters=True)
# Train!
logger.info("***** Running training *****")
logger.info(" Num Epochs = %d", args.num_train_epochs)
logger.info(" Instantaneous batch size per GPU = %d",
args.train_batch_size)
logger.info(
" Total train batch size (w. parallel, distributed & accumulation) = %d",
args.train_batch_size * args.gradient_accumulation_steps *
(torch.distributed.get_world_size() if args.local_rank != -1 else 1))
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Total optimization steps = %d", t_total)
global_step = 0
best_acc = 0
model.zero_grad()
seed_everything(args.seed)
for epoch in range(int(args.num_train_epochs)):
tr_loss = AverageMeter()
pbar = ProgressBar(n_total=len(train_dataloader), desc='Training')
for step, batch in enumerate(train_dataloader):
model.train()
batch = tuple(t.to(args.device) for t in batch)
inputs = {
'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[3]
}
inputs['token_type_ids'] = batch[2]
outputs = model(**inputs)
loss = outputs[
0] # model outputs are always tuple in transformers (see doc)
if args.n_gpu > 1:
loss = loss.mean(
) # mean() to average on multi-gpu parallel training
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer),
args.max_grad_norm)
else:
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
tr_loss.update(loss.item(), n=1)
pbar(step, info={"loss": loss.item()})
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step() # Update learning rate schedule
model.zero_grad()
global_step += 1
train_log = {'loss': tr_loss.avg}
eval_log = evaluate(args, model, eval_dataloader, metrics)
logs = dict(train_log, **eval_log)
show_info = f'\nEpoch: {epoch} - ' + "-".join(
[f' {key}: {value:.4f} ' for key, value in logs.items()])
logger.info(show_info)
if logs['eval_acc'] > best_acc:
logger.info(
f"\nEpoch {epoch}: eval_acc improved from {best_acc} to {logs['eval_acc']}"
)
logger.info("save model to disk.")
best_acc = logs['eval_acc']
print("Valid Entity Score: ")
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
output_file = args.model_save_path
output_file.mkdir(exist_ok=True)
output_model_file = output_file / WEIGHTS_NAME
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = output_file / CONFIG_NAME
with open(str(output_config_file), 'w') as f:
f.write(model_to_save.config.to_json_string())
def evaluate(args, model, eval_dataloader, metrics):
# Eval!
logger.info(" Number of examples = %d", len(eval_dataloader))
logger.info(" Batch size = %d", args.eval_batch_size)
eval_loss = AverageMeter()
metrics.reset()
preds = []
targets = []
pbar = ProgressBar(n_total=len(eval_dataloader), desc='Evaluating')
# pdb.set_trace()
# (Pdb) a
# args = Namespace(adam_epsilon=1e-08, albert_config_path=
# 'pretrain/pytorch/albert_base_zh/albert_config_base.json',
# arch='albert_base', bert_dir='pretrain/pytorch/albert_base_zh',
# device=device(type='cuda'), do_eval=False, do_lower_case=False,
# do_test=True, do_train=False, eval_all_checkpoints=False,
# eval_batch_size=16, eval_max_seq_len=64, evaluate_during_training=False,
# fp16=False, fp16_opt_level='O1', gradient_accumulation_steps=1, learning_rate=2e-05,
# local_rank=-1, max_grad_norm=5.0, model_save_path=PosixPath('outputs/checkpoints/albert_base'),
# n_gpu=1, no_cuda=False, num_train_epochs=3.0, overwrite_cache=False,
# overwrite_output_dir=False, seed=42, server_ip='', server_port='',
# share_type='all', task_name='lcqmc', train_batch_size=32, train_max_seq_len=64,
# warmup_proportion=0.1, weight_decay=0.1)
#
# model = AlbertForSequenceClassification(
# (bert): AlbertModel(
# (embeddings): AlbertEmbeddings(
# (word_embeddings): Embedding(21128, 128, padding_idx=0)
# (word_embeddings_2): Linear(in_features=128, out_features=768, bias=False)
# (position_embeddings): Embedding(512, 768)
# (token_type_embeddings): Embedding(2, 768)
# (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
# (dropout): Dropout(p=0.0, inplace=False)
# )
# (encoder): AlbertEncoder(
# (layer_shared): AlbertLayer(
# (attention): AlbertAttention(
# (self): AlbertSelfAttention(
# (query): Linear(in_features=768, out_features=768, bias=True)
# (key): Linear(in_features=768, out_features=768, bias=True)
# (value): Linear(in_features=768, out_features=768, bias=True)
# (dropout): Dropout(p=0.0, inplace=False)
# )
# (output): AlbertSelfOutput(
# (dense): Linear(in_features=768, out_features=768, bias=True)
# (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
# (dropout): Dropout(p=0.0, inplace=False)
# )
# )
# (intermediate): AlbertIntermediate(
# (dense): Linear(in_features=768, out_features=3072, bias=True)
# )
# (output): AlbertOutput(
# (dense): Linear(in_features=3072, out_features=768, bias=True)
# (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
# (dropout): Dropout(p=0.0, inplace=False)
# )
# )
# )
# (pooler): AlbertPooler(
# (dense): Linear(in_features=768, out_features=768, bias=True)
# (activation): Tanh()
# )
# )
# (dropout): Dropout(p=0.2, inplace=False)
# (classifier): Linear(in_features=768, out_features=2, bias=True)
# )
# eval_dataloader = <torch.utils.data.dataloader.DataLoader object at 0x7f113f07d668>
# metrics = <common.metrics.Accuracy object at 0x7f11a904fa90>
for bid, batch in enumerate(eval_dataloader):
model.eval()
batch = tuple(t.to(args.device) for t in batch)
with torch.no_grad():
inputs = {
'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2],
'labels': batch[3]
}
# inputs['token_type_ids'] = batch[2]
outputs = model(**inputs)
loss, logits = outputs[:2]
eval_loss.update(loss.item(), n=batch[0].size()[0])
preds.append(logits.cpu().detach())
targets.append(inputs['labels'].cpu().detach())
pbar(bid)
# pdb.set_trace()
# (Pdb) pp batch[0].size(), batch[1].size(), batch[2].size(), batch[3].size()
# (torch.Size([16, 64]), torch.Size([16, 64]), torch.Size([16, 64]), torch.Size([16]))
# (Pdb) inputs['input_ids'][0]
# tensor([ 101, 6443, 3300, 4312, 676, 6821, 2476, 7770, 3926, 4638, 102, 6821,
# 2476, 7770, 3926, 1745, 8024, 6443, 3300, 102, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0], device='cuda:0')
# (Pdb) inputs['attention_mask'][0]
# tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], device='cuda:0')
# (Pdb) inputs['token_type_ids'][0]
# tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], device='cuda:0')
# (Pdb) inputs['labels']
# tensor([0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1], device='cuda:0')
preds = torch.cat(preds, dim=0).cpu().detach()
targets = torch.cat(targets, dim=0).cpu().detach()
metrics(preds, targets)
eval_log = {"eval_acc": metrics.value(), 'eval_loss': eval_loss.avg}
return eval_log
def create_features(self, examples, max_seq_len, cached_features_file):
'''
# The convention in BERT is:
# (a) For sequence pairs:
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
# (b) For single sequences:
# tokens: [CLS] the dog is hairy . [SEP]
# type_ids: 0 0 0 0 0 0 0
'''
pbar = ProgressBar(n_total=len(examples), desc='create features')
if cached_features_file.exists():
logger.info("Loading features from cached file %s",
cached_features_file)
features = torch.load(cached_features_file)
else:
features = []
for ex_id, example in enumerate(examples):
tokens_a = self.tokenizer.tokenize(example.text_a)
tokens_b = None
label_id = example.label
if example.text_b:
tokens_b = self.tokenizer.tokenize(example.text_b)
# Modifies `tokens_a` and `tokens_b` in place so that the total
# length is less than the specified length.
# Account for [CLS], [SEP], [SEP] with "- 3"
self.truncate_seq_pair(tokens_a,
tokens_b,
max_length=max_seq_len - 3)
else:
# Account for [CLS] and [SEP] with '-2'
if len(tokens_a) > max_seq_len - 2:
tokens_a = tokens_a[:max_seq_len - 2]
tokens = ['[CLS]'] + tokens_a + ['[SEP]']
segment_ids = [0] * len(tokens)
if tokens_b:
tokens += tokens_b + ['[SEP]']
segment_ids += [1] * (len(tokens_b) + 1)
input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
input_mask = [1] * len(input_ids)
padding = [0] * (max_seq_len - len(input_ids))
input_len = len(input_ids)
input_ids += padding
input_mask += padding
segment_ids += padding
assert len(input_ids) == max_seq_len
assert len(input_mask) == max_seq_len
assert len(segment_ids) == max_seq_len
if ex_id < 2:
logger.info("*** Example ***")
logger.info(f"guid: {example.guid}" % ())
logger.info(
f"tokens: {' '.join([str(x) for x in tokens])}")
logger.info(
f"input_ids: {' '.join([str(x) for x in input_ids])}")
logger.info(
f"input_mask: {' '.join([str(x) for x in input_mask])}"
)
logger.info(
f"segment_ids: {' '.join([str(x) for x in segment_ids])}"
)
logger.info(f"label id : {label_id}")
feature = InputFeature(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id,
input_len=input_len)
features.append(feature)
pbar(step=ex_id)
logger.info("Saving features into cached file %s",
cached_features_file)
torch.save(features, cached_features_file)
return features
def __init__(self,
training_path,
file_id,
tokenizer,
data_name,
reduce_memory=False):
self.tokenizer = tokenizer
self.file_id = file_id
data_file = training_path / f"{data_name}_file_{self.file_id}.json"
metrics_file = training_path / f"{data_name}_file_{self.file_id}_metrics.json"
assert data_file.is_file() and metrics_file.is_file()
metrics = json.loads(metrics_file.read_text())
num_samples = metrics['num_training_examples']
seq_len = metrics['max_seq_len']
self.temp_dir = None
self.working_dir = None
if reduce_memory:
self.temp_dir = TemporaryDirectory()
self.working_dir = Path(self.temp_dir.name)
input_ids = np.memmap(filename=self.working_dir /
'input_ids.memmap',
mode='w+',
dtype=np.int32,
shape=(num_samples, seq_len))
input_masks = np.memmap(filename=self.working_dir /
'input_masks.memmap',
shape=(num_samples, seq_len),
mode='w+',
dtype=np.bool)
segment_ids = np.memmap(filename=self.working_dir /
'segment_ids.memmap',
shape=(num_samples, seq_len),
mode='w+',
dtype=np.bool)
lm_label_ids = np.memmap(filename=self.working_dir /
'lm_label_ids.memmap',
shape=(num_samples, seq_len),
mode='w+',
dtype=np.int32)
lm_label_ids[:] = -1
is_nexts = np.memmap(filename=self.working_dir / 'is_nexts.memmap',
shape=(num_samples, ),
mode='w+',
dtype=np.bool)
else:
input_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.int32)
input_masks = np.zeros(shape=(num_samples, seq_len), dtype=np.bool)
segment_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.bool)
lm_label_ids = np.full(shape=(num_samples, seq_len),
dtype=np.int32,
fill_value=-1)
is_nexts = np.zeros(shape=(num_samples, ), dtype=np.bool)
logger.info(f"Loading training examples for {str(data_file)}")
with data_file.open() as f:
for i, line in enumerate(f):
line = line.strip()
example = json.loads(line)
features = convert_example_to_features(example, tokenizer,
seq_len)
input_ids[i] = features.input_ids
segment_ids[i] = features.segment_ids
input_masks[i] = features.input_mask
lm_label_ids[i] = features.lm_label_ids
is_nexts[i] = features.is_next
assert i == num_samples - 1 # Assert that the sample count metric was true
logger.info("Loading complete!")
self.num_samples = num_samples
self.seq_len = seq_len
self.input_ids = input_ids
self.input_masks = input_masks
self.segment_ids = segment_ids
self.lm_label_ids = lm_label_ids
self.is_nexts = is_nexts
def main():
parser = ArgumentParser()
parser.add_argument('--data_name', default='albert', type=str)
parser.add_argument(
"--file_num",
type=int,
default=10,
help="Number of dynamic masking to pregenerate (with different masks)")
parser.add_argument(
"--reduce_memory",
action="store_true",
help=
"Store training data as on-disc memmaps to massively reduce memory usage"
)
parser.add_argument("--epochs",
type=int,
default=4,
help="Number of epochs to train for")
parser.add_argument('--share_type',
default='all',
type=str,
choices=['all', 'attention', 'ffn', 'None'])
parser.add_argument('--num_eval_steps', default=100)
parser.add_argument('--num_save_steps', default=200)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument("--train_batch_size",
default=4,
type=int,
help="Total batch size for training.")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument("--warmup_proportion",
default=0.1,
type=float,
help="Linear warmup over warmup_steps.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument('--max_grad_norm', default=1.0, type=float)
parser.add_argument("--learning_rate",
default=0.00176,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--fp16_opt_level',
type=str,
default='O2',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
args = parser.parse_args()
pregenerated_data = config['data_dir'] / "corpus/train"
assert pregenerated_data.is_dir(), \
"--pregenerated_data should point to the folder of files made by prepare_lm_data_mask.py!"
samples_per_epoch = 0
for i in range(args.file_num):
data_file = pregenerated_data / f"{args.data_name}_file_{i}.json"
metrics_file = pregenerated_data / f"{args.data_name}_file_{i}_metrics.json"
if data_file.is_file() and metrics_file.is_file():
metrics = json.loads(metrics_file.read_text())
samples_per_epoch += metrics['num_training_examples']
else:
if i == 0:
exit("No training data was found!")
print(
f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs})."
)
print(
"This script will loop over the available data, but training diversity may be negatively impacted."
)
break
logger.info(f"samples_per_epoch: {samples_per_epoch}")
if args.local_rank == -1 or args.no_cuda:
device = torch.device(f"cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
args.n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
f"device: {device} , distributed training: {bool(args.local_rank != -1)}, 16-bits training: {args.fp16}, "
f"share_type: {args.share_type}")
if args.gradient_accumulation_steps < 1:
raise ValueError(
f"Invalid gradient_accumulation_steps parameter: {args.gradient_accumulation_steps}, should be >= 1"
)
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
seed_everything(args.seed)
tokenizer = BertTokenizer(vocab_file=config['albert_vocab_path'])
total_train_examples = samples_per_epoch * args.epochs
num_train_optimization_steps = int(total_train_examples /
args.train_batch_size /
args.gradient_accumulation_steps)
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
args.warmup_steps = int(num_train_optimization_steps *
args.warmup_proportion)
bert_config = BertConfig.from_pretrained(str(config['albert_config_path']),
share_type=args.share_type)
model = BertForPreTraining(config=bert_config)
# model = BertForMaskedLM.from_pretrained(config['checkpoint_dir'] / 'checkpoint-580000')
model.to(device)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
# optimizer = Lamb(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
lr_scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=args.warmup_steps,
t_total=num_train_optimization_steps)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
global_step = 0
mask_metric = LMAccuracy()
sop_metric = LMAccuracy()
tr_mask_acc = AverageMeter()
tr_sop_acc = AverageMeter()
tr_loss = AverageMeter()
tr_mask_loss = AverageMeter()
tr_sop_loss = AverageMeter()
loss_fct = CrossEntropyLoss(ignore_index=-1)
train_logs = {}
logger.info("***** Running training *****")
logger.info(f" Num examples = {total_train_examples}")
logger.info(f" Batch size = {args.train_batch_size}")
logger.info(f" Num steps = {num_train_optimization_steps}")
logger.info(f" warmup_steps = {args.warmup_steps}")
start_time = time.time()
seed_everything(args.seed) # Added here for reproducibility
for epoch in range(args.epochs):
for idx in range(args.file_num):
epoch_dataset = PregeneratedDataset(
file_id=idx,
training_path=pregenerated_data,
tokenizer=tokenizer,
reduce_memory=args.reduce_memory,
data_name=args.data_name)
if args.local_rank == -1:
train_sampler = RandomSampler(epoch_dataset)
else:
train_sampler = DistributedSampler(epoch_dataset)
train_dataloader = DataLoader(epoch_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, lm_label_ids, is_next = batch
outputs = model(input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask)
prediction_scores = outputs[0]
seq_relationship_score = outputs[1]
masked_lm_loss = loss_fct(
prediction_scores.view(-1, bert_config.vocab_size),
lm_label_ids.view(-1))
next_sentence_loss = loss_fct(
seq_relationship_score.view(-1, 2), is_next.view(-1))
loss = masked_lm_loss + next_sentence_loss
mask_metric(logits=prediction_scores.view(
-1, bert_config.vocab_size),
target=lm_label_ids.view(-1))
sop_metric(logits=seq_relationship_score.view(-1, 2),
target=is_next.view(-1))
if args.n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
nb_tr_steps += 1
tr_mask_acc.update(mask_metric.value(), n=input_ids.size(0))
tr_sop_acc.update(sop_metric.value(), n=input_ids.size(0))
tr_loss.update(loss.item(), n=1)
tr_mask_loss.update(masked_lm_loss.item(), n=1)
tr_sop_loss.update(next_sentence_loss.item(), n=1)
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
torch.nn.utils.clip_grad_norm_(
amp.master_params(optimizer), args.max_grad_norm)
else:
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
lr_scheduler.step()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if global_step % args.num_eval_steps == 0:
now = time.time()
eta = now - start_time
if eta > 3600:
eta_format = ('%d:%02d:%02d' %
(eta // 3600,
(eta % 3600) // 60, eta % 60))
elif eta > 60:
eta_format = '%d:%02d' % (eta // 60, eta % 60)
else:
eta_format = '%ds' % eta
train_logs['loss'] = tr_loss.avg
train_logs['mask_acc'] = tr_mask_acc.avg
train_logs['sop_acc'] = tr_sop_acc.avg
train_logs['mask_loss'] = tr_mask_loss.avg
train_logs['sop_loss'] = tr_sop_loss.avg
show_info = f'[Training]:[{epoch}/{args.epochs}]{global_step}/{num_train_optimization_steps} ' \
f'- ETA: {eta_format}' + "-".join(
[f' {key}: {value:.4f} ' for key, value in train_logs.items()])
logger.info(show_info)
tr_mask_acc.reset()
tr_sop_acc.reset()
tr_loss.reset()
tr_mask_loss.reset()
tr_sop_loss.reset()
start_time = now
if global_step % args.num_save_steps == 0:
if args.local_rank in [-1, 0] and args.num_save_steps > 0:
# Save model checkpoint
output_dir = config[
'checkpoint_dir'] / f'lm-checkpoint-{global_step}'
if not output_dir.exists():
output_dir.mkdir()
# save model
model_to_save = model.module if hasattr(
model, 'module'
) else model # Take care of distributed/parallel training
model_to_save.save_pretrained(str(output_dir))
torch.save(args, str(output_dir / 'training_args.bin'))
logger.info("Saving model checkpoint to %s",
output_dir)
# save config
output_config_file = output_dir / CONFIG_NAME
with open(str(output_config_file), 'w') as f:
f.write(model_to_save.config.to_json_string())
# save vocab
tokenizer.save_vocabulary(output_dir)
def main():
parser = ArgumentParser()
parser.add_argument('--data_name', default='albert', type=str)
parser.add_argument('--max_ngram', default=3, type=int)
parser.add_argument("--do_data", default=False, action='store_true')
parser.add_argument("--do_split", default=False, action='store_true')
parser.add_argument("--do_lower_case", default=False, action='store_true')
parser.add_argument('--seed', default=42, type=int)
parser.add_argument("--line_per_file", default=1000000000, type=int)
parser.add_argument(
"--file_num",
type=int,
default=10,
help="Number of dynamic masking to pregenerate (with different masks)")
parser.add_argument("--max_seq_len", type=int, default=128)
parser.add_argument(
"--short_seq_prob",
type=float,
default=0.1,
help="Probability of making a short sentence as a training example")
parser.add_argument(
"--masked_lm_prob",
type=float,
default=0.15,
help="Probability of masking each token for the LM task")
parser.add_argument(
"--max_predictions_per_seq",
type=int,
default=20, # 128 * 0.15
help="Maximum number of tokens to mask in each sequence")
args = parser.parse_args()
seed_everything(args.seed)
logger.info("pregenerate training data parameters:\n %s", args)
tokenizer = BertTokenizer(vocab_file=config['data_dir'] / 'vocab.txt',
do_lower_case=args.do_lower_case)
if args.do_split:
corpus_path = config['data_dir'] / "corpus/corpus.txt"
split_save_path = config['data_dir'] / "corpus/train"
if not split_save_path.exists():
split_save_path.mkdir(exist_ok=True)
line_per_file = args.line_per_file
command = f'split -a 4 -l {line_per_file} -d {corpus_path} {split_save_path}/shard_'
os.system(f"{command}")
if args.do_data:
data_path = config['data_dir'] / "corpus/train"
files = sorted([
f for f in data_path.parent.iterdir()
if f.exists() and '.txt' in str(f)
])
for idx in range(args.file_num):
logger.info(f"pregenetate {args.data_name}_file_{idx}.json")
save_filename = data_path / f"{args.data_name}_file_{idx}.json"
num_instances = 0
with save_filename.open('w') as fw:
for file_idx in range(len(files)):
file_path = files[file_idx]
file_examples = create_training_instances(
input_file=file_path,
tokenizer=tokenizer,
max_seq_len=args.max_seq_len,
max_ngram=args.max_ngram,
short_seq_prob=args.short_seq_prob,
masked_lm_prob=args.masked_lm_prob,
max_predictions_per_seq=args.max_predictions_per_seq)
file_examples = [
json.dumps(instance) for instance in file_examples
]
for instance in file_examples:
fw.write(instance + '\n')
num_instances += 1
metrics_file = data_path / f"{args.data_name}_file_{idx}_metrics.json"
print(f"num_instances: {num_instances}")
with metrics_file.open('w') as metrics_file:
metrics = {
"num_training_examples": num_instances,
"max_seq_len": args.max_seq_len
}
metrics_file.write(json.dumps(metrics))
def create_training_instances(input_file, tokenizer, max_seq_len,
short_seq_prob, max_ngram, masked_lm_prob,
max_predictions_per_seq):
"""Create `TrainingInstance`s from raw text."""
all_documents = [[]]
# Input file format:
# (1) One sentence per line. These should ideally be actual sentences, not
# entire paragraphs or arbitrary spans of text. (Because we use the
# sentence boundaries for the "next sentence prediction" task).
# (2) Blank lines between documents. Document boundaries are needed so
# that the "next sentence prediction" task doesn't span between documents.
f = open(input_file, 'r')
lines = f.readlines()
pbar = ProgressBar(n_total=len(lines), desc='read data')
for line_cnt, line in enumerate(lines):
line = line.strip()
# Empty lines are used as document delimiters
if not line:
all_documents.append([])
tokens = tokenizer.tokenize(line)
if tokens:
all_documents[-1].append(tokens)
pbar(step=line_cnt)
print(' ')
# Remove empty documents
all_documents = [x for x in all_documents if x]
random.shuffle(all_documents)
vocab_words = list(tokenizer.vocab.keys())
instances = []
pbar = ProgressBar(n_total=len(all_documents), desc='create instances')
for document_index in range(len(all_documents)):
instances.extend(
create_instances_from_document(all_documents, document_index,
max_seq_len, short_seq_prob,
max_ngram, masked_lm_prob,
max_predictions_per_seq,
vocab_words))
pbar(step=document_index)
print(' ')
ex_idx = 0
while ex_idx < 5:
instance = instances[ex_idx]
logger.info("-------------------------Example-----------------------")
logger.info(f"id: {ex_idx}")
logger.info(
f"tokens: {' '.join([str(x) for x in instance['tokens']])}")
logger.info(
f"masked_lm_labels: {' '.join([str(x) for x in instance['masked_lm_labels']])}"
)
logger.info(
f"segment_ids: {' '.join([str(x) for x in instance['segment_ids']])}"
)
logger.info(
f"masked_lm_positions: {' '.join([str(x) for x in instance['masked_lm_positions']])}"
)
logger.info(f"is_random_next : {instance['is_random_next']}")
ex_idx += 1
random.shuffle(instances)
return instances
def main():
parser = argparse.ArgumentParser()
# parser.add_argument("--arch", default='albert_xlarge', type=str)
# parser.add_argument('--task_name', default='lcqmc', type=str)
parser.add_argument(
"--train_max_seq_len",
default=64,
type=int,
help=
"The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.")
parser.add_argument(
"--eval_max_seq_len",
default=64,
type=int,
help=
"The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.")
parser.add_argument('--share_type',
default='all',
type=str,
choices=['all', 'attention', 'ffn', 'None'])
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_test",
action='store_true',
help="Whether to run eval on the test set.")
# parser.add_argument("--evaluate_during_training", action='store_true',
# help="Rul evaluation during training at each logging step.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--eval_batch_size",
default=16,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument("--learning_rate",
default=2e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.1,
type=float,
help="Weight deay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=5.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=int,
help=
"Proportion of training to perform linear learning rate warmup for,E.g., 0.1 = 10% of training."
)
parser.add_argument(
"--eval_all_checkpoints",
action='store_true',
help=
"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number"
)
parser.add_argument("--no_cuda",
action='store_true',
help="Avoid using CUDA when available")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument(
'--overwrite_cache',
action='store_true',
help="Overwrite the cached training and evaluation sets")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--fp16',
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"
)
parser.add_argument(
'--fp16_opt_level',
type=str,
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument('--server_ip',
type=str,
default='',
help="For distant debugging.")
parser.add_argument('--server_port',
type=str,
default='',
help="For distant debugging.")
parser.add_argument('--train_data_num',
type=int,
default=None,
help="Use a small number to test the full code")
parser.add_argument('--eval_data_num',
type=int,
default=None,
help="Use a small number to test the full code")
parser.add_argument('--do_pred',
action='store_true',
help="Predict a dataset and do not evaluate accuracy")
parser.add_argument(
'--model_size',
type=str,
default='large',
help=
"Which albert size to choose, could be: base, large, xlarge, xxlarge")
parser.add_argument('--commit',
type=str,
default='',
help="Current experiment's commit")
parser.add_argument(
'--load_checkpoints_dir',
type=str,
default="",
help=
"Whether to use checkpoints to load model. If not given checkpoints, use un-fineturned albert"
)
args = parser.parse_args()
config = create_config(commit=args.commit,
model_size=args.model_size,
load_checkpoints_dir=args.load_checkpoints_dir)
# args.model_save_path = config['checkpoints_dir']
# args.model_save_path.mkdir()
# os.makedirs(config["checkpoints_dir"])
os.makedirs(config["output_dir"])
with open(config["args"], "w") as fa, open(config["config"], "w") as fc:
json.dump(vars(args), fa, indent=4)
json.dump({k: str(v) for k, v in config.items()}, fc, indent=4)
# Setudistant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
args.device = device
init_logger(log_file=config['log'])
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank, device, args.n_gpu, bool(args.local_rank != -1),
args.fp16)
# Set seed
seed_everything(args.seed)
# --------- data
processor = BertProcessor(vocab_path=config['albert_vocab_path'],
do_lower_case=args.do_lower_case)
label_list = processor.get_labels()
num_labels = len(label_list)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
bert_config = BertConfig.from_pretrained(str(config['bert_dir'] /
'config.json'),
share_type=args.share_type,
num_labels=num_labels)
logger.info("Training/evaluation parameters %s", args)
metrics = Accuracy(topK=1)
# Training
if args.do_train:
# train_data = processor.get_train(config['data_dir'] / "train.tsv")
# train_examples = processor.create_examples(lines=train_data, example_type='train',
# cached_examples_file=config[
# 'data_dir'] / f"cached_train_examples_{args.arch}")
# todo: 划分数据集，合成train.csv, eval.csv, test. csv
train_examples = processor.read_data_and_create_examples(
example_type='train',
cached_examples_file=config['data_dir'] /
f"cached_train_examples_{args.model_size}",
input_file=config['data_dir'] / "train.csv")
train_examples = train_examples[:args.
train_data_num] if args.train_data_num is not None else train_examples
train_features = processor.create_features(
examples=train_examples,
max_seq_len=args.train_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_train_features_{}_{}".format(args.train_max_seq_len,
args.model_size))
train_features = train_features[:args.
train_data_num] if args.train_data_num is not None else train_features
train_dataset = processor.create_dataset(train_features)
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
train_sampler_eval = SequentialSampler(train_dataset)
train_dataloader_eval = DataLoader(train_dataset,
sampler=train_sampler_eval,
batch_size=args.eval_batch_size)
# valid_data = processor.get_dev(config['data_dir'] / "dev.tsv")
# valid_examples = processor.create_examples(lines=valid_data, example_type='valid',
# cached_examples_file=config[
# 'data_dir'] / f"cached_valid_examples_{args.arch}")
valid_examples = processor.read_data_and_create_examples(
example_type='valid',
cached_examples_file=config['data_dir'] /
f"cached_valid_examples_{args.model_size}",
input_file=config['data_dir'] / "valid.csv")
valid_examples = valid_examples[:args.
eval_data_num] if args.eval_data_num is not None else valid_examples
valid_features = processor.create_features(
examples=valid_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_valid_features_{}_{}".format(args.eval_max_seq_len,
args.model_size))
valid_features = valid_features[:args.
eval_data_num] if args.eval_data_num is not None else valid_features
valid_dataset = processor.create_dataset(valid_features)
valid_sampler = SequentialSampler(valid_dataset)
valid_dataloader = DataLoader(valid_dataset,
sampler=valid_sampler,
batch_size=args.eval_batch_size)
model = BertForSequenceClassification.from_pretrained(
config['bert_dir'], config=bert_config)
if args.local_rank == 0:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
model.to(args.device)
train(args, train_dataloader, valid_dataloader, train_dataloader_eval,
metrics, model, config)
# 打上戳表示训练完成
config["success_train"].open("w").write("Train Success!!")
# if args.do_test:
# model = BertForSequenceClassification.from_pretrained(args.model_save_path, config=bert_config)
# test_data = processor.get_train(config['data_dir'] / "test.tsv")
# test_examples = processor.create_examples(lines=test_data,
# example_type='test',
# cached_examples_file=config[
# 'data_dir'] / f"cached_test_examples_{args.arch}")
# test_features = processor.create_features(examples=test_examples,
# max_seq_len=args.eval_max_seq_len,
# cached_features_file=config[
# 'data_dir'] / "cached_test_features_{}_{}".format(
# args.eval_max_seq_len, args.arch
# ))
# test_dataset = processor.create_dataset(test_features)
# test_sampler = SequentialSampler(test_dataset)
# test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=args.eval_batch_size)
# # model = BertForSequenceClassification.from_pretrained(args.model_save_path, config=bert_config)
# model.to(args.device)
# test_log = evaluate(args, model, test_dataloader, metrics)
# print(test_log)
if args.do_pred:
pred_examples = processor.read_data_and_create_examples(
example_type='predict',
cached_examples_file=config['data_dir'] /
f"cached_pred_examples_{args.model_size}",
input_file=config['data_dir'] / "pred.csv")
pred_examples = pred_examples[:args.
eval_data_num] if args.eval_data_num is not None else pred_examples
pred_features = processor.create_features(
examples=pred_examples,
max_seq_len=args.eval_max_seq_len,
cached_features_file=config['data_dir'] /
"cached_pred_features_{}_{}".format(args.eval_max_seq_len,
args.model_size))
pred_features = pred_features[:args.
eval_data_num] if args.eval_data_num is not None else pred_features
pred_dataset = processor.create_dataset(pred_features)
pred_sampler = SequentialSampler(pred_dataset)
pred_dataloader = DataLoader(pred_dataset,
sampler=pred_sampler,
batch_size=args.eval_batch_size)
param_dir = config['checkpoints_dir'] if (config["checkpoints_dir"] / "pytorch_model.bin").exists() \
else config['bert_dir']
model = BertForSequenceClassification.from_pretrained(
param_dir, config=bert_config)
model.to(args.device)
# todo
predict(args, model, pred_dataloader, config)
# 打上戳表示预测完成
config["success_predict"].open("w").write("Predict Success!!")
config["output_dir"].rename(config["output_dir"].parent /
("success-" + config["output_dir"].name))
def create_features(self, examples, max_seq_len, cached_features_file):
'''
# The convention in ALBERT is:
# (a) For sequence pairs:
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
# (b) For single sequences:
# tokens: [CLS] the dog is hairy . [SEP]
# type_ids: 0 0 0 0 0 0 0
'''
pbar = ProgressBar(n_total=len(examples), desc='create features')
# pdb.set_trace()
# cached_features_file = PosixPath('dataset/lcqmc/cached_test_features_64_albert_base')
if cached_features_file.exists():
logger.info("Loading features from cached file %s",
cached_features_file)
features = torch.load(cached_features_file)
else:
features = []
for ex_id, example in enumerate(examples):
tokens_a = self.tokenizer.tokenize(example.text_a)
tokens_b = None
label_id = example.label
if example.text_b:
tokens_b = self.tokenizer.tokenize(example.text_b)
# Modifies `tokens_a` and `tokens_b` in place so that the total
# length is less than the specified length.
# Account for [CLS], [SEP], [SEP] with "- 3"
self.truncate_seq_pair(tokens_a,
tokens_b,
max_length=max_seq_len - 3)
else:
# Account for [CLS] and [SEP] with '-2'
if len(tokens_a) > max_seq_len - 2:
tokens_a = tokens_a[:max_seq_len - 2]
tokens = ['[CLS]'] + tokens_a + ['[SEP]']
segment_ids = [0] * len(tokens)
if tokens_b:
tokens += tokens_b + ['[SEP]']
segment_ids += [1] * (len(tokens_b) + 1)
input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
# pdb.set_trace()
# (Pdb) pp example.text_a, example.text_b, example.label
# ('谁有狂三这张高清的', '这张高清图，谁有', 0)
# (Pdb) pp tokens_a, tokens_b
# (['谁', '有', '狂', '三', '这', '张', '高', '清', '的'],
# ['这', '张', '高', '清', '图', '，', '谁', '有'])
# (Pdb) input_ids
# [101, 6443, 3300, 4312, 676, 6821, 2476, 7770, 3926, 4638, 102,
# 6821, 2476, 7770, 3926, 1745, 8024, 6443, 3300, 102]
input_mask = [1] * len(input_ids)
padding = [0] * (max_seq_len - len(input_ids))
input_len = len(input_ids)
input_ids += padding
input_mask += padding
segment_ids += padding
assert len(input_ids) == max_seq_len
assert len(input_mask) == max_seq_len
assert len(segment_ids) == max_seq_len
if ex_id < 2:
logger.info("*** Example ***")
logger.info(f"guid: {example.guid}" % ())
logger.info(
f"tokens: {' '.join([str(x) for x in tokens])}")
logger.info(
f"input_ids: {' '.join([str(x) for x in input_ids])}")
logger.info(
f"input_mask: {' '.join([str(x) for x in input_mask])}"
)
logger.info(
f"segment_ids: {' '.join([str(x) for x in segment_ids])}"
)
logger.info(f"label id : {label_id}")
feature = InputFeature(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id,
input_len=input_len)
features.append(feature)
pbar(step=ex_id)
logger.info("Saving features into cached file %s",
cached_features_file)
torch.save(features, cached_features_file)
return features
