random.seed(args.seed)

np.random.seed(args.seed)

torch.manual_seed(args.seed)

if args.n_gpu > 0:

""" Train the model """

args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)

train_sampler = RandomSampler(train_dataset)

train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)

if args.max_steps > 0:

t_total = args.max_steps

args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1

else:

t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs

warm_up_steps=int(args.warmup_steps*t_total)

logging_steps=int(args.logging_steps*t_total)

save_steps=int(args.save_steps*t_total)

# Prepare optimizer and schedule (linear warmup and decay)

no_decay = ['bias', 'LayerNorm.weight']

optimizer_grouped_parameters = [

{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},

{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

]

optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warm_up_steps, t_total=t_total)

# multi-gpu training (should be after apex fp16 initialization)

if args.n_gpu > 1:

model = torch.nn.DataParallel(model)

# Train!

logger.info("***** Running training *****")

logger.info(" Num examples = %d", len(train_dataset))

logger.info(" Num Epochs = %d", args.num_train_epochs)

logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)

logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps)

logger.info(" Total optimization steps = %d", t_total)

global_step = 0

tr_loss, logging_loss = 0.0, 0.0

max_acc=0

max_f1=0

model.zero_grad()

train_iterator = trange(int(args.num_train_epochs), desc="Epoch")

set_seed(args) # Added here for reproductibility (even between python 2 and 3)

for _ in train_iterator:

epoch_iterator = tqdm(train_dataloader, desc="Iteration")

for step, batch in enumerate(epoch_iterator):

model.train()

batch = tuple(t.to(args.device) for t in batch)

inputs = {'input_ids': batch[0],

'attention_mask': batch[1],

'align_mask': batch[2],

'labels': batch[4]}

inputs['token_type_ids'] = batch[3]

outputs = model(**inputs)

loss = outputs[0] # model outputs are always tuple in transformers (see doc)

if args.n_gpu > 1:

loss = loss.mean() # mean() to average on multi-gpu parallel training

if args.gradient_accumulation_steps > 1:

loss = loss / args.gradient_accumulation_steps

loss.backward()

torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

tr_loss += loss.item()

if (step + 1) % args.gradient_accumulation_steps == 0:

optimizer.step()

scheduler.step() # Update learning rate schedule

model.zero_grad()

global_step += 1

if logging_steps > 0 and global_step % logging_steps == 0:

if args.evaluate_during_training:

results = evaluate(args, model, tokenizer)

for key, value in results.items():

if key=="acc":

max_acc=max([max_acc,value])

with open(os.path.join(args.output_dir, "acc.txt"), 'a+') as w:

w.write("%d\t%f\t%f\n" % (global_step, value, max_acc))

if key == "f1":

max_f1=max([max_f1,value])

with open(os.path.join(args.output_dir, "f1.txt"), 'a+') as w:

w.write("%d\t%f\t%f\n" % (global_step, value, max_f1))

with open(os.path.join(args.output_dir, "loss.txt"), 'a+') as w:

w.write("%d\t%f\n"%(global_step, (tr_loss - logging_loss) / logging_steps))

logging_loss = tr_loss

if save_steps > 0 and global_step % save_steps == 0:

# Save model checkpoint

output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))

if not os.path.exists(output_dir):

os.makedirs(output_dir)

model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training

model_to_save.save_pretrained(output_dir)

torch.save(args, os.path.join(output_dir, 'training_args.bin'))

logger.info("Saving model checkpoint to %s", output_dir)

if args.max_steps > 0 and global_step > args.max_steps:

epoch_iterator.close()

break

if args.max_steps > 0 and global_step > args.max_steps:

train_iterator.close()

break

# Loop to handle MNLI double evaluation (matched, mis-matched)

eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,)

eval_outputs_dirs = (args.output_dir, args.output_dir + '-MM') if args.task_name == "mnli" else (args.output_dir,)

results = {}

for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):

eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)

if not os.path.exists(eval_output_dir):

os.makedirs(eval_output_dir)

args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)

# Note that DistributedSampler samples randomly

eval_sampler = SequentialSampler(eval_dataset)

eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)

# Eval!

logger.info("***** Running evaluation {} *****".format(prefix))

logger.info(" Num examples = %d", len(eval_dataset))

logger.info(" Batch size = %d", args.eval_batch_size)

eval_loss = 0.0

nb_eval_steps = 0

preds = None

out_label_ids = None

for batch in tqdm(eval_dataloader, desc="Evaluating"):

model.eval()

batch = tuple(t.to(args.device) for t in batch)

with torch.no_grad():

inputs = {'input_ids': batch[0],

'attention_mask': batch[1],

'align_mask':batch[2],

'labels': batch[4]}

inputs['token_type_ids'] = batch[3]

outputs = model(**inputs)

tmp_eval_loss, logits = outputs[:2]

eval_loss += tmp_eval_loss.mean().item()

nb_eval_steps += 1

if preds is None:

preds = logits.detach().cpu().numpy()

out_label_ids = inputs['labels'].detach().cpu().numpy()

else:

preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)

out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)

eval_loss = eval_loss / nb_eval_steps

if args.output_mode == "classification":

preds = np.argmax(preds, axis=1)

elif args.output_mode == "regression":

preds = np.squeeze(preds)

result = compute_metrics(eval_task, preds, out_label_ids)

results.update(result)

output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")

with open(output_eval_file, "a+") as writer:

logger.info("***** Eval results {} *****".format(prefix))

for key in sorted(result.keys()):

logger.info(" %s = %s", key, str(result[key]))

writer.write("%s = %s\n" % (key, str(result[key])))

processor = processors[task]()

output_mode = output_modes[task]

# Load data features from cache or dataset file

cached_features_file = os.path.join(args.data_dir, 'cached_{}_bert_align_{}_{}'.format(

'dev' if evaluate else 'train',

str(args.max_seq_length),

str(task)))

if os.path.exists(cached_features_file):

logger.info("Loading features from cached file %s", cached_features_file)

features = torch.load(cached_features_file)

else:

logger.info("Creating features from dataset file at %s", args.data_dir)

label_list = processor.get_labels()

examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)

features = convert_examples_to_features(examples,

tokenizer,

label_list=label_list,

max_length=args.max_seq_length,

output_mode=output_mode,

pad_on_left=False, # pad on the left for xlnet

pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],

pad_token_segment_id=0,

)

logger.info("Saving features into cached file %s", cached_features_file)

torch.save(features, cached_features_file)

# Convert to Tensors and build dataset

all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)

all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)

all_align_mask = torch.tensor([f.align_mask for f in features], dtype=torch.long)

all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)

if output_mode == "classification":

all_labels = torch.tensor([f.label for f in features], dtype=torch.long)

elif output_mode == "regression":

all_labels = torch.tensor([f.label for f in features], dtype=torch.float)

dataset = TensorDataset(all_input_ids, all_attention_mask,all_align_mask,all_token_type_ids, all_labels)

parser = argparse.ArgumentParser()

## Required parameters

parser.add_argument("--data_dir", default="data/QNLI", type=str,

help="The input data dir. Should contain the .tsv files (or other data files) for the task.")

parser.add_argument("--task_name", default="QNLI", type=str,

help="The name of the task to train selected in the list: " + ", ".join(processors.keys()))

parser.add_argument("--output_dir", default="output_qnli", type=str,

help="The output directory where the model predictions and checkpoints will be written.")

## Other parameters

parser.add_argument("--max_seq_length", default=95, type=int,

help="The maximum total input sequence length after tokenization. Sequences longer "

"than this will be truncated, sequences shorter will be padded.")

parser.add_argument("--do_train", default=True,

help="Whether to run training.")

parser.add_argument("--do_eval", default=True,

help="Whether to run eval on the dev set.")

parser.add_argument("--evaluate_during_training", default=True,

help="Rul evaluation during training at each logging step.")

parser.add_argument("--do_lower_case", action='store_true',

help="Set this flag if you are using an uncased model.")

parser.add_argument("--per_gpu_train_batch_size", default=64, type=int,

help="Batch size per GPU/CPU for training.")

parser.add_argument("--per_gpu_eval_batch_size", default=96, type=int,

help="Batch size per GPU/CPU for evaluation.")

parser.add_argument('--gradient_accumulation_steps', type=int, default=1,

help="Number of updates steps to accumulate before performing a backward/update pass.")

parser.add_argument("--learning_rate", default=3e-5, type=float,

help="The initial learning rate for Adam.")

parser.add_argument("--weight_decay", default=1e-5, type=float,

help="Weight deay if we apply some.")

parser.add_argument("--adam_epsilon", default=1e-8, type=float,

help="Epsilon for Adam optimizer.")

parser.add_argument("--max_grad_norm", default=5.0, type=float,

help="Max gradient norm.")

parser.add_argument("--num_train_epochs", default=3.0, type=float,

help="Total number of training epochs to perform.")

parser.add_argument("--max_steps", default=-1, type=int,

help="If > 0: set total number of training steps to perform. Override num_train_epochs.")

parser.add_argument("--warmup_steps", default=0.1, type=float,

help="Linear warmup over warmup_steps.")

parser.add_argument('--logging_steps', type=float, default=0.1,

help="Log every X updates steps.")

parser.add_argument('--save_steps', type=float, default=0.1,

help="Save checkpoint every X updates steps.")

parser.add_argument("--eval_all_checkpoints", default=False,

help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")

parser.add_argument('--overwrite_output_dir', default=True,

help="Overwrite the content of the output directory")

parser.add_argument('--seed', type=int, default=42,

help="random seed for initialization")

args = parser.parse_args()

if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:

raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))

# Setup CUDA, GPU & distributed training

device = torch.device("cuda")

args.n_gpu = torch.cuda.device_count()

args.device = device

# Setup logging

logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',

datefmt = '%m/%d/%Y %H:%M:%S',

level = logging.INFO)

logger.warning("device: %s, n_gpu: %s",device, args.n_gpu)

# Set seed

set_seed(args)

# Prepare GLUE task

args.task_name = args.task_name.lower()

if args.task_name not in processors:

raise ValueError("Task not found: %s" % (args.task_name))

processor = processors[args.task_name]()

args.output_mode = output_modes[args.task_name]

label_list = processor.get_labels()

num_labels = len(label_list)

# Load pretrained model and tokenizer

config_class, model_class, tokenizer_class = BertConfig,BertForSequenceClassification,BertTokenizer

config = config_class.from_pretrained("pretrained/cased_L-12_H-768_A-12/bert_config.json",num_labels=num_labels,finetuning_task=args.task_name)

tokenizer = tokenizer_class.from_pretrained("pretrained/cased_L-12_H-768_A-12/vocab.txt")

model = model_class.from_pretrained("pretrained/bert-base.pt", from_tf=False, config=config)

model.to(args.device)

logger.info("Training/evaluation parameters %s", args)

# Training

if args.do_train:

train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)

global_step, tr_loss = train(args, train_dataset, model, tokenizer)

logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)

# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()

if args.do_train:

# Create output directory if needed

if not os.path.exists(args.output_dir):

os.makedirs(args.output_dir)

logger.info("Saving model checkpoint to %s", args.output_dir)

# Save a trained model, configuration and tokenizer using `save_pretrained()`.

# They can then be reloaded using `from_pretrained()`

model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training

model_to_save.save_pretrained(args.output_dir)

tokenizer.save_pretrained(args.output_dir)

# Good practice: save your training arguments together with the trained model

torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))

# Load a trained model and vocabulary that you have fine-tuned

model = model_class.from_pretrained(args.output_dir)

tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)

model.to(args.device)

# Evaluation

results = {}

if args.do_eval:

tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)

checkpoints = [args.output_dir]

if args.eval_all_checkpoints:

checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))

logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging

logger.info("Evaluate the following checkpoints: %s", checkpoints)

for checkpoint in checkpoints:

global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""

model = model_class.from_pretrained(checkpoint)

model.to(args.device)

result = evaluate(args, model, tokenizer, prefix=global_step)

result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())

results.update(result)