def test_full_tokenizer(self):
tokenizer = RobertaTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map)
text = "lower newer"
bpe_tokens = ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
tokens = tokenizer.tokenize(text)
self.assertListEqual(tokens, bpe_tokens)
input_tokens = tokens + [tokenizer.unk_token]
input_bpe_tokens = [14, 15, 10, 9, 3, 2, 15, 19]
self.assertListEqual(
tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens)
def test_full_tokenizer(self):
tokenizer = RobertaTokenizer(self.vocab_file, self.merges_file,
**self.special_tokens_map)
text = "lower"
bpe_tokens = ["low", "er"]
tokens = tokenizer.tokenize(text)
self.assertListEqual(tokens, bpe_tokens)
input_tokens = tokens + [tokenizer.unk_token]
input_bpe_tokens = [13, 12, 17]
self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens),
input_bpe_tokens)
def __init__(self, params, shared=None):
super(CrossEncoderRanker, self).__init__()
self.params = params
self.device = torch.device("cuda" if torch.cuda.is_available()
and not params["no_cuda"] else "cpu")
self.n_gpu = torch.cuda.device_count()
if params.get("roberta"):
self.tokenizer = RobertaTokenizer.from_pretrained(
params["bert_model"], )
else:
self.tokenizer = BertTokenizer.from_pretrained(
params["bert_model"], do_lower_case=params["lowercase"])
special_tokens_dict = {
"additional_special_tokens": [
ENT_START_TAG,
ENT_END_TAG,
ENT_TITLE_TAG,
],
}
self.tokenizer.add_special_tokens(special_tokens_dict)
self.NULL_IDX = self.tokenizer.pad_token_id
self.START_TOKEN = self.tokenizer.cls_token
self.END_TOKEN = self.tokenizer.sep_token
# init model
self.build_model()
if params["path_to_model"] is not None:
self.load_model(params["path_to_model"])
self.model = self.model.to(self.device)
self.data_parallel = params.get("data_parallel")
if self.data_parallel:
self.model = torch.nn.DataParallel(self.model)
def main():
tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=True)
reader = RoBertaMCQParallelScoreReader(debug=True)
out = reader.read("dummy_data.jsonl", tokenizer, 70, None)
print(len(out))
tokens, segs, masks,scores, labels = out[0]
print(tokens.size())
print(segs)
print(masks)
print(scores)
print(labels.size()) # shoud be 0
def LoadDatasetEval(args, config, task_cfg, task_id):
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
else:
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
task = "TASK" + task_id
task_name = task_cfg[task]["name"]
# initialize the feature reader
feats_h5path1 = task_cfg[task]["features_h5path1"]
feats_h5path2 = task_cfg[task]["features_h5path2"]
features_reader1 = ImageFeaturesH5Reader(feats_h5path1, config, args.in_memory) if feats_h5path1 != "" else None
features_reader2 = ImageFeaturesH5Reader(feats_h5path2, config, args.in_memory) if feats_h5path2 != "" else None
batch_size = task_cfg[task].get("eval_batch_size", args.batch_size)
if args.local_rank != -1:
batch_size = int(batch_size / dist.get_world_size())
logger.info("Loading %s Dataset with batch size %d" % (task_name, batch_size))
if args.split:
eval_split = args.split
else:
eval_split = task_cfg[task]["val_split"]
dset_val = DatasetMapEval[task_name](
task=task_cfg[task]["name"],
dataroot=task_cfg[task]["dataroot"],
annotations_jsonpath=task_cfg[task]["val_annotations_jsonpath"],
split=eval_split,
image_features_reader=features_reader1,
gt_image_features_reader=features_reader2,
tokenizer=tokenizer,
bert_model=args.bert_model,
padding_index=0,
max_seq_length=task_cfg[task]["max_seq_length"],
max_region_num=task_cfg[task]["max_region_num"],
num_locs=config.num_locs,
add_global_imgfeat=config.add_global_imgfeat,
append_mask_sep=(config.fusion_method == 'vl-bert_vqa'),
)
dl_val = DataLoader(
dset_val,
shuffle=False,
batch_size=batch_size,
num_workers=10,
pin_memory=True,
drop_last=args.drop_last,
)
task2num_iters = {task: len(dl_val)}
return batch_size, task2num_iters, dset_val, dl_val
def test_sequence_builders(self):
tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
text = tokenizer.encode("sequence builders")
text_2 = tokenizer.encode("multi-sequence build")
encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True)
encoded_pair_from_decode = tokenizer.encode("sequence builders", "multi-sequence build", add_special_tokens=True)
encoded_sentence = tokenizer.add_special_tokens_single_sentence(text)
encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2)
assert encoded_sentence == encoded_text_from_decode
assert encoded_pair == encoded_pair_from_decode
def __init__(self, params):
super(CrossEncoderRanker, self).__init__()
self.params = params
self.device = torch.device(
"cuda" if torch.cuda.is_available() and not params["no_cuda"] else "cpu"
)
self.n_gpu = torch.cuda.device_count()
if params.get("roberta"):
self.tokenizer = RobertaTokenizer.from_pretrained(params["bert_model"], do_lower_case=params["lowercase"])
else:
self.tokenizer = BertTokenizer.from_pretrained(
params["bert_model"], do_lower_case=params["lowercase"]
)
special_tokens_dict = {
"additional_special_tokens": [
ENT_START_TAG,
ENT_END_TAG,
ENT_TITLE_TAG,
],
}
self.tokenizer.add_special_tokens(special_tokens_dict)
self.NULL_IDX = self.tokenizer.pad_token_id
self.START_TOKEN = self.tokenizer.cls_token
self.END_TOKEN = self.tokenizer.sep_token
self.START_MENTION_ID = self.tokenizer.convert_tokens_to_ids(ENT_START_TAG)
self.END_MENTION_ID = self.tokenizer.convert_tokens_to_ids(ENT_END_TAG)
# keep some parameters around
self.add_sigmoid = params["add_sigmoid"]
self.margin = params["margin"]
self.objective = params["objective"]
self.pos_neg_loss = params.get("pos_neg_loss", False)
assert self.objective == "softmax" or self.objective == "max_margin"
# init model
self.build_model()
if params["path_to_model"] is not None:
self.load_model(params["path_to_model"])
self.model = self.model.to(self.device)
self.data_parallel = params.get("data_parallel")
if self.data_parallel:
self.model = torch.nn.DataParallel(self.model)
def LoadDatasets(args, task_cfg, ids, split="trainval"):
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case
)
else:
tokenizer = BertTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case
)
task_feature_reader1 = {}
task_feature_reader2 = {}
for i, task_id in enumerate(ids):
task = "TASK" + task_id
if task_cfg[task]["features_h5path1"] not in task_feature_reader1:
task_feature_reader1[task_cfg[task]["features_h5path1"]] = None
if task_cfg[task]["features_h5path2"] not in task_feature_reader2:
task_feature_reader2[task_cfg[task]["features_h5path2"]] = None
# initilzie the feature reader
for features_h5path in task_feature_reader1.keys():
if features_h5path != "":
task_feature_reader1[features_h5path] = ImageFeaturesH5Reader(
features_h5path, args.in_memory
)
for features_h5path in task_feature_reader2.keys():
if features_h5path != "":
task_feature_reader2[features_h5path] = ImageFeaturesH5Reader(
features_h5path, args.in_memory
)
task_datasets_train = {}
task_datasets_val = {}
task_dataloader_train = {}
task_dataloader_val = {}
task_ids = []
task_batch_size = {}
task_num_iters = {}
for i, task_id in enumerate(ids):
task = "TASK" + task_id
task_name = task_cfg[task]["name"]
task_ids.append(task)
batch_size = task_cfg[task]["batch_size"] // args.gradient_accumulation_steps
num_workers = args.num_workers
if args.local_rank != -1:
batch_size = int(batch_size / dist.get_world_size())
num_workers = int(num_workers / dist.get_world_size())
# num_workers = int(num_workers / len(ids))
logger.info(
"Loading %s Dataset with batch size %d"
% (task_cfg[task]["name"], batch_size)
)
task_datasets_train[task] = None
if "train" in split:
task_datasets_train[task] = DatasetMapTrain[task_name](
task=task_cfg[task]["name"],
dataroot=task_cfg[task]["dataroot"],
annotations_jsonpath=task_cfg[task]["train_annotations_jsonpath"],
split=task_cfg[task]["train_split"],
image_features_reader=task_feature_reader1[
task_cfg[task]["features_h5path1"]
],
gt_image_features_reader=task_feature_reader2[
task_cfg[task]["features_h5path2"]
],
tokenizer=tokenizer,
bert_model=args.bert_model,
clean_datasets=args.clean_train_sets,
padding_index=0,
max_seq_length=task_cfg[task]["max_seq_length"],
max_region_num=task_cfg[task]["max_region_num"],
)
task_datasets_val[task] = None
if "val" in split:
task_datasets_val[task] = DatasetMapTrain[task_name](
task=task_cfg[task]["name"],
dataroot=task_cfg[task]["dataroot"],
annotations_jsonpath=task_cfg[task]["val_annotations_jsonpath"],
split=task_cfg[task]["val_split"],
image_features_reader=task_feature_reader1[
task_cfg[task]["features_h5path1"]
],
gt_image_features_reader=task_feature_reader2[
task_cfg[task]["features_h5path2"]
],
tokenizer=tokenizer,
bert_model=args.bert_model,
clean_datasets=args.clean_train_sets,
padding_index=0,
max_seq_length=task_cfg[task]["max_seq_length"],
max_region_num=task_cfg[task]["max_region_num"],
)
task_num_iters[task] = 0
task_batch_size[task] = 0
if "train" in split:
if args.local_rank == -1:
train_sampler = RandomSampler(task_datasets_train[task])
else:
# TODO: check if this works with current data generator from disk that relies on next(file)
# (it doesn't return item back by index)
train_sampler = DistributedSampler(task_datasets_train[task])
task_dataloader_train[task] = DataLoader(
task_datasets_train[task],
sampler=train_sampler,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=True,
)
task_num_iters[task] = len(task_dataloader_train[task])
task_batch_size[task] = batch_size
if "val" in split:
task_dataloader_val[task] = DataLoader(
task_datasets_val[task],
shuffle=False,
batch_size=batch_size,
num_workers=2,
pin_memory=True,
)
return (
task_batch_size,
task_num_iters,
task_ids,
task_datasets_train,
task_datasets_val,
task_dataloader_train,
task_dataloader_val,
)
import pickle
import argparse
from pytorch_transformers.tokenization_roberta import RobertaTokenizer
from mspan_roberta_gcn.drop_roberta_dataset import DropReader
from tag_mspan_robert_gcn.drop_roberta_mspan_dataset import DropReader as TDropReader
parser = argparse.ArgumentParser()
parser.add_argument("--input_path", type=str)
parser.add_argument("--output_dir", type=str)
parser.add_argument("--passage_length_limit", type=int, default=463)
parser.add_argument("--question_length_limit", type=int, default=46)
parser.add_argument("--tag_mspan", action="store_true")
args = parser.parse_args()
tokenizer = RobertaTokenizer.from_pretrained(args.input_path +
"/roberta.large")
if args.tag_mspan:
dev_reader = TDropReader(tokenizer, args.passage_length_limit,
args.question_length_limit)
train_reader = TDropReader(tokenizer,
args.passage_length_limit,
args.question_length_limit,
skip_when_all_empty=[
"passage_span", "question_span",
"addition_subtraction", "counting",
"multi_span"
])
data_format = "drop_dataset_{}.json"
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--file_path",
default="data/conceptual_caption/",
type=str,
help="The input train corpus.",
)
parser.add_argument(
"--from_pretrained",
default="bert-base-uncased",
type=str,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-base-uncased, roberta-base, roberta-large, ",
)
parser.add_argument(
"--bert_model",
default="bert-base-uncased",
type=str,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, roberta-base",
)
parser.add_argument(
"--output_dir",
default="save",
type=str,
# required=True,
help=
"The output directory where the model checkpoints will be written.",
)
parser.add_argument(
"--config_file",
type=str,
default="config/bert_base_6layer_6conect.json",
help="The config file which specified the model details.",
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=36,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.",
)
parser.add_argument(
"--train_batch_size",
default=512,
type=int,
help="Total batch size for training.",
)
parser.add_argument(
"--learning_rate",
default=1e-4,
type=float,
help="The initial learning rate for Adam.",
)
parser.add_argument(
"--num_train_epochs",
default=10.0,
type=float,
help="Total number of training epochs to perform.",
)
parser.add_argument(
"--start_epoch",
default=0,
type=float,
help="Total number of training epochs to perform.",
)
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.",
)
parser.add_argument("--img_weight",
default=1,
type=float,
help="weight for image loss")
parser.add_argument("--no_cuda",
action="store_true",
help="Whether not to use CUDA when available")
parser.add_argument(
"--on_memory",
action="store_true",
help="Whether to load train samples into memory or use disk",
)
parser.add_argument(
"--do_lower_case",
type=bool,
default=True,
help=
"Whether to lower case the input text. True for uncased models, False for cased models.",
)
parser.add_argument(
"--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus",
)
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumualte before performing a backward/update pass.",
)
parser.add_argument(
"--fp16",
action="store_true",
help="Whether to use 16-bit float precision instead of 32-bit",
)
parser.add_argument(
"--loss_scale",
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n",
)
parser.add_argument(
"--dynamic_attention",
action="store_true",
help="whether use dynamic attention.",
)
parser.add_argument(
"--num_workers",
type=int,
default=25,
help="Number of workers in the dataloader.",
)
parser.add_argument("--save_name",
default="",
type=str,
help="save name for training.")
parser.add_argument(
"--baseline",
action="store_true",
help="Wheter to use the baseline model (single bert).",
)
parser.add_argument(
"--freeze",
default=-1,
type=int,
help="till which layer of textual stream of vilbert need to fixed.",
)
parser.add_argument(
"--distributed",
action="store_true",
help="whether use chunck for parallel training.",
)
parser.add_argument("--without_coattention",
action="store_true",
help="whether pair loss.")
parser.add_argument(
"--visual_target",
default=0,
type=int,
help="which target to use for visual branch. \
0: soft label, \
1: regress the feature, \
2: NCE loss.",
)
parser.add_argument(
"--objective",
default=0,
type=int,
help="which objective to use \
0: with ICA loss, \
1: with ICA loss, for the not aligned pair, no masking objective, \
2: without ICA loss, do not sample negative pair.",
)
parser.add_argument("--num_negative",
default=255,
type=int,
help="num of negative to use")
parser.add_argument("--resume_file",
default="",
type=str,
help="Resume from checkpoint")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
args = parser.parse_args()
if args.baseline:
from pytorch_pretrained_bert.modeling import BertConfig
from vilbert.basebert import BertForMultiModalPreTraining
else:
from vilbert.vilbert import BertForMultiModalPreTraining, BertConfig
if args.save_name:
prefix = "-" + args.save_name
else:
prefix = ""
timeStamp = args.config_file.split("/")[1].split(".")[0] + prefix
savePath = os.path.join(args.output_dir, timeStamp)
bert_weight_name = json.load(
open("config/" + args.from_pretrained + "_weight_name.json", "r"))
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend="nccl")
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
default_gpu = False
if dist.is_available() and args.local_rank != -1:
rank = dist.get_rank()
if rank == 0:
default_gpu = True
else:
default_gpu = True
if default_gpu:
if not os.path.exists(savePath):
os.makedirs(savePath)
config = BertConfig.from_json_file(args.config_file)
if default_gpu:
# save all the hidden parameters.
with open(os.path.join(savePath, "command.txt"), "w") as f:
print(args, file=f) # Python 3.x
print("\n", file=f)
print(config, file=f)
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
cache = 5000
if dist.is_available() and args.local_rank != -1:
num_replicas = dist.get_world_size()
args.train_batch_size = args.train_batch_size // num_replicas
args.num_workers = args.num_workers // num_replicas
cache = cache // num_replicas
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case)
else:
tokenizer = BertTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case)
num_train_optimization_steps = None
train_dataset = ConceptCapLoaderTrain(
args.file_path,
tokenizer,
args.bert_model,
seq_len=args.max_seq_length,
batch_size=args.train_batch_size,
visual_target=args.visual_target,
num_workers=args.num_workers,
local_rank=args.local_rank,
objective=args.objective,
cache=cache,
)
validation_dataset = ConceptCapLoaderVal(
args.file_path,
tokenizer,
args.bert_model,
seq_len=args.max_seq_length,
batch_size=args.train_batch_size,
visual_target=args.visual_target,
num_workers=2,
objective=args.objective,
)
num_train_optimization_steps = int(
train_dataset.num_dataset / args.train_batch_size /
args.gradient_accumulation_steps) * (args.num_train_epochs -
args.start_epoch)
task_names = ["Conceptual_Caption"]
task_ids = ["TASK0"]
task_num_iters = {
"TASK0": train_dataset.num_dataset / args.train_batch_size
}
logdir = os.path.join("logs", timeStamp)
if default_gpu:
tbLogger = utils.tbLogger(
logdir,
savePath,
task_names,
task_ids,
task_num_iters,
args.gradient_accumulation_steps,
)
if args.visual_target == 0:
config.v_target_size = 1601
config.visual_target = args.visual_target
else:
config.v_target_size = 2048
config.visual_target = args.visual_target
if "roberta" in args.bert_model:
config.model = "roberta"
if args.freeze > config.t_biattention_id[0]:
config.fixed_t_layer = config.t_biattention_id[0]
if args.without_coattention:
config.with_coattention = False
if args.dynamic_attention:
config.dynamic_attention = True
if args.from_pretrained:
model = BertForMultiModalPreTraining.from_pretrained(
args.from_pretrained, config=config, default_gpu=default_gpu)
else:
model = BertForMultiModalPreTraining(config)
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
if args.freeze != -1:
bert_weight_name_filtered = []
for name in bert_weight_name:
if "embeddings" in name:
bert_weight_name_filtered.append(name)
elif "encoder" in name:
layer_num = name.split(".")[2]
if int(layer_num) <= args.freeze:
bert_weight_name_filtered.append(name)
optimizer_grouped_parameters = []
for key, value in dict(model.named_parameters()).items():
if key[12:] in bert_weight_name_filtered:
value.requires_grad = False
if default_gpu:
print("filtered weight")
print(bert_weight_name_filtered)
if not args.from_pretrained:
param_optimizer = list(model.named_parameters())
optimizer_grouped_parameters = [
{
"params": [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
"weight_decay":
0.01,
},
{
"params": [
p for n, p in param_optimizer
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
else:
optimizer_grouped_parameters = []
for key, value in dict(model.named_parameters()).items():
if value.requires_grad:
if key[12:] in bert_weight_name:
lr = args.learning_rate * 0.1
else:
lr = args.learning_rate
if any(nd in key for nd in no_decay):
optimizer_grouped_parameters += [{
"params": [value],
"lr": lr,
"weight_decay": 0.0
}]
if not any(nd in key for nd in no_decay):
optimizer_grouped_parameters += [{
"params": [value],
"lr": lr,
"weight_decay": 0.01
}]
if default_gpu:
print(len(list(model.named_parameters())),
len(optimizer_grouped_parameters))
# set different parameters for vision branch and lanugage branch.
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
optimizer = FusedAdam(
optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0,
)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer,
static_loss_scale=args.loss_scale)
else:
optimizer = AdamW(
optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon,
betas=(0.9, 0.98),
)
scheduler = WarmupLinearSchedule(
optimizer,
warmup_steps=args.warmup_proportion * num_train_optimization_steps,
t_total=num_train_optimization_steps,
)
startIterID = 0
global_step = 0
if args.resume_file != "" and os.path.exists(args.resume_file):
checkpoint = torch.load(args.resume_file, map_location="cpu")
new_dict = {}
for attr in checkpoint["model_state_dict"]:
if attr.startswith("module."):
new_dict[attr.replace(
"module.", "", 1)] = checkpoint["model_state_dict"][attr]
else:
new_dict[attr] = checkpoint["model_state_dict"][attr]
model.load_state_dict(new_dict)
scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
global_step = checkpoint["global_step"]
del checkpoint
model.cuda()
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = v.cuda()
if args.fp16:
model.half()
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
if default_gpu:
logger.info("***** Running training *****")
logger.info(" Num examples = %d", train_dataset.num_dataset)
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
for epochId in range(int(args.start_epoch), int(args.num_train_epochs)):
model.train()
for step, batch in enumerate(train_dataset):
iterId = startIterID + step + (epochId * len(train_dataset))
image_ids = batch[-1]
batch = tuple(
t.cuda(device=device, non_blocking=True) for t in batch[:-1])
input_ids, input_mask, segment_ids, lm_label_ids, is_next, image_feat, image_loc, image_target, image_label, image_mask = (
batch)
if args.objective == 1:
image_label = image_label * (is_next == 0).long().unsqueeze(1)
image_label[image_label == 0] = -1
lm_label_ids = lm_label_ids * (is_next
== 0).long().unsqueeze(1)
lm_label_ids[lm_label_ids == 0] = -1
masked_loss_t, masked_loss_v, next_sentence_loss = model(
input_ids,
image_feat,
image_loc,
segment_ids,
input_mask,
image_mask,
lm_label_ids,
image_label,
image_target,
is_next,
)
if args.objective == 2:
next_sentence_loss = next_sentence_loss * 0
masked_loss_v = masked_loss_v * args.img_weight
loss = masked_loss_t + masked_loss_v + next_sentence_loss
if n_gpu > 1:
loss = loss.mean()
masked_loss_t = masked_loss_t.mean()
masked_loss_v = masked_loss_v.mean()
next_sentence_loss = next_sentence_loss.mean()
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
lr_this_step = args.learning_rate * warmup_linear(
global_step / num_train_optimization_steps,
args.warmup_proportion,
)
for param_group in optimizer.param_groups:
param_group["lr"] = lr_this_step
scheduler.step()
optimizer.step()
optimizer.zero_grad()
global_step += 1
if default_gpu:
tbLogger.step_train_CC(
epochId,
iterId,
float(masked_loss_t),
float(masked_loss_v),
float(next_sentence_loss),
optimizer.param_groups[0]["lr"],
"TASK0",
"train",
)
if (step % (20 * args.gradient_accumulation_steps) == 0
and step != 0 and default_gpu):
tbLogger.showLossTrainCC()
# Do the evaluation
torch.set_grad_enabled(False)
numBatches = len(validation_dataset)
model.eval()
for step, batch in enumerate(validation_dataset):
image_ids = batch[-1]
batch = tuple(
t.cuda(device=device, non_blocking=True) for t in batch[:-1])
input_ids, input_mask, segment_ids, lm_label_ids, is_next, image_feat, image_loc, image_target, image_label, image_mask = (
batch)
batch_size = input_ids.size(0)
masked_loss_t, masked_loss_v, next_sentence_loss = model(
input_ids,
image_feat,
image_loc,
segment_ids,
input_mask,
image_mask,
lm_label_ids,
image_label,
image_target,
is_next,
)
masked_loss_v = masked_loss_v * args.img_weight
loss = masked_loss_t + masked_loss_v + next_sentence_loss
if n_gpu > 1:
loss = loss.mean()
masked_loss_t = masked_loss_t.mean()
masked_loss_v = masked_loss_v.mean()
next_sentence_loss = next_sentence_loss.mean()
if default_gpu:
tbLogger.step_val_CC(
epochId,
float(masked_loss_t),
float(masked_loss_v),
float(next_sentence_loss),
"TASK0",
batch_size,
"val",
)
sys.stdout.write("%d / %d \r" % (step, numBatches))
sys.stdout.flush()
if default_gpu:
ave_score = tbLogger.showLossValCC()
torch.set_grad_enabled(True)
if default_gpu:
# Save a trained model
logger.info("** ** * Saving fine - tuned model ** ** * ")
model_to_save = (
model.module if hasattr(model, "module") else model
) # Only save the model it-self
output_model_file = os.path.join(
savePath, "pytorch_model_" + str(epochId) + ".bin")
output_checkpoint = os.path.join(
savePath, "pytorch_ckpt_" + str(epochId) + ".tar")
torch.save(model_to_save.state_dict(), output_model_file)
torch.save(
{
"model_state_dict": model_to_save.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"global_step": global_step,
},
output_checkpoint,
)
if default_gpu:
tbLogger.txt_close()
from pytorch_transformers import BertTokenizer
from mspan_roberta_gcn.drop_roberta_dataset import DropReader
from tag_mspan_robert_gcn.drop_roberta_mspan_dataset import DropReader as TDropReader
parser = argparse.ArgumentParser()
parser.add_argument("--input_path", type=str)
parser.add_argument("--model_path", type=str)
parser.add_argument("--output_dir", type=str)
parser.add_argument("--passage_length_limit", type=int, default=463)
parser.add_argument("--question_length_limit", type=int, default=46)
parser.add_argument("--tag_mspan", action="store_true")
parser.add_argument("--eng", type=int, default=1)
args = parser.parse_args()
if args.eng != 0:
tokenizer = RobertaTokenizer.from_pretrained(args.model_path)
else:
# import pdb; pdb.set_trace()
tokenizer = BertTokenizer.from_pretrained(args.model_path)
if args.tag_mspan:
dev_reader = TDropReader(tokenizer,
args.passage_length_limit,
args.question_length_limit,
is_eng=args.eng)
train_reader = TDropReader(tokenizer,
args.passage_length_limit,
args.question_length_limit,
skip_when_all_empty=[
"passage_span", "question_span",
def get_tokenizer(self, **kwargs):
kwargs.update(self.special_tokens_map)
return RobertaTokenizer.from_pretrained(self.tmpdirname, **kwargs)
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--training_data_path",
default=None,
type=str,
required=True,
help="The training data path")
parser.add_argument("--validation_data_path",
default=None,
type=str,
required=True,
help="The validation data path")
parser.add_argument(
"--mcq_model",
default=None,
type=str,
required=True,
help="choose one from the list: bert-mcq-parallel-max, "
"bert-mcq_parallel-weighted-sum, bert-mcq-concat, mac-bert, or add roberta instead of bert"
)
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese, roberta-base, roberta-large"
)
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=32,
type=int,
help="Total batch size for eval.")
parser.add_argument("--learning_rate",
default=2e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--warmup_proportion",
default=0.1,
type=float,
help=
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument(
'--loss_scale',
type=float,
default=0,
help=
"Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
parser.add_argument("--max_grad_norm",
default=None,
type=float,
help="Max gradient norm.")
parser.add_argument(
'--fp16_opt_level',
type=str,
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight deay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--dropout", default=0.0, type=float, help="dropout")
parser.add_argument(
"--eval_freq",
default=0,
type=int,
help="Evaluation steps frequency. Default is at the end of each epoch. "
"You can also increase the frequency")
parser.add_argument(
'--tie_weights_weighted_sum',
action='store_true',
help="Whether to tie the weights for the weighted sum model")
parser.add_argument('--max_number_premises',
type=int,
default=None,
help="Number of premise sentences to use at max")
parser.add_argument('--num_labels',
type=int,
default=3,
help="Number of labels")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument('--with_score',
action='store_true',
help="Knowledge with score is provided")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
logger.info(
"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".
format(device, n_gpu, bool(args.local_rank != -1), args.fp16))
# true batch size
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
if args.gradient_accumulation_steps < 1:
raise ValueError(
"Invalid gradient_accumulation_steps parameter: {}, should be >= 1"
.format(args.gradient_accumulation_steps))
if not args.do_train and not args.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and not args.overwrite_output_dir:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
with open(os.path.join(args.output_dir, "mcq_inputs.json"), 'w') as f:
json.dump(vars(args), f, indent=2)
stdout_handler = prepare_global_logging(args.output_dir, False)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(
"roberta-large", do_lower_case=args.do_lower_case)
logger.info("Type of Tokenizer : ROBERTA")
else:
tokenizer = BertTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case)
logger.info("Type of Tokenizer : BERT")
data_reader = None
if args.mcq_model == 'bert-mcq-parallel-max':
model = BertMCQParallel.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-concat':
model = BertMCQConcat.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQConcatReader()
elif args.mcq_model == 'bert-mcq-weighted-sum':
model = BertMCQWeightedSum.from_pretrained(
args.bert_model,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-simple-sum':
model = BertMCQSimpleSum.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-mac':
model = BertMCQMAC.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-parallel-max':
model = RoBertaMCQParallel.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-concat':
model = RoBertaMCQConcat.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQConcatReader()
elif args.mcq_model == 'roberta-mcq-weighted-sum':
model = RoBertaMCQWeightedSum.from_pretrained(
args.bert_model,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-ws-score':
model = RoBertaMCQWeightedSumScore.from_pretrained(
args.bert_model,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelScoreReader()
elif args.mcq_model == 'roberta-mcq-simple-sum':
model = RoBertaMCQSimpleSum.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-ss-score':
model = RoBertaMCQSimpleSumScore.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelScoreReader()
elif args.mcq_model == 'roberta-mcq-mac':
model = RoBertaMCQMAC.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-conv3d':
model = RoBertaMCQConv3d.from_pretrained(
args.bert_model,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
else:
logger.error(f"Invalid MCQ model name {args.mcq_model}")
exit(0)
if args.do_train:
# Prepare data loader
# get data loader for train/dev
train_data = data_reader.read(args.training_data_path, tokenizer,
args.max_seq_length,
args.max_number_premises)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
eval_data = data_reader.read(args.validation_data_path, tokenizer,
args.max_seq_length,
args.max_number_premises)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
# num_train_optimization_steps, dividing by effective batch size
t_total = (len(train_dataloader) //
args.gradient_accumulation_steps) * args.num_train_epochs
num_train_optimization_steps = (
len(train_dataloader) //
args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size(
)
# Prepare optimizer
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay':
args.weight_decay
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay':
0.0
}]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=args.warmup_steps,
t_total=t_total)
model.to(device)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1 and not args.no_cuda:
model = torch.nn.DataParallel(model)
global_step = 0
number_of_batches_per_epoch = len(train_dataloader)
if args.eval_freq > 0:
steps_to_eval = args.eval_freq
else:
steps_to_eval = number_of_batches_per_epoch
logger.info("***** Training *****")
logger.info(" num examples = %d", len(train_data))
logger.info(" batch size = %d", args.train_batch_size)
logger.info(" num steps = %d", num_train_optimization_steps)
logger.info(" number of Gpus= %d", n_gpu)
logger.info("***** Evaluation *****")
logger.info(" num examples = %d", len(eval_data))
logger.info(" batch size = %d", args.eval_batch_size)
best_acc = 0.0
best_epoch = 1
for epoch_index in trange(int(args.num_train_epochs), desc="Epoch"):
epoch_start_time = time.time()
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
tq = tqdm(train_dataloader, desc="Iteration")
acc = 0
for step, batch in enumerate(tq):
batch = tuple(t.to(device) for t in batch)
if not args.with_score:
input_ids, segment_ids, input_mask, label_ids = batch
outputs = model(input_ids, segment_ids, input_mask,
label_ids)
else:
input_ids, segment_ids, input_mask, scores, label_ids = batch
outputs = model(input_ids, segment_ids, input_mask, scores,
label_ids)
loss = outputs[0]
logits = outputs[1]
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_accuracy = accuracy(logits, label_ids)
acc += tmp_accuracy
if n_gpu > 1:
loss = loss.mean(
) # mean() to average on multi-gpu parallel training
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
if args.max_grad_norm is not None:
torch.nn.utils.clip_grad_norm_(
amp.master_params(optimizer), args.max_grad_norm)
else:
loss.backward()
if args.max_grad_norm is not None:
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
if (step + 1) % args.gradient_accumulation_steps == 0:
scheduler.step() # Update learning rate schedule
optimizer.step()
model.zero_grad()
global_step += 1
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
tq.set_description(
_get_loss_accuracy(tr_loss / nb_tr_steps,
acc / nb_tr_examples))
# TODO: always eval on last batch
# For now select the batch_size appropriately
if (((step + 1) % steps_to_eval == 0) or (step+1)==number_of_batches_per_epoch )\
and args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
etq = tqdm(eval_dataloader, desc="Validating")
for batch in etq:
batch = tuple(t.to(device) for t in batch)
with torch.no_grad():
if not args.with_score:
input_ids, segment_ids, input_mask, label_ids = batch
outputs = model(input_ids, segment_ids,
input_mask, label_ids)
else:
input_ids, segment_ids, input_mask, scores, label_ids = batch
outputs = model(input_ids, segment_ids,
input_mask, scores, label_ids)
tmp_eval_loss = outputs[0]
logits = outputs[1]
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
etq.set_description(
_get_loss_accuracy(
eval_loss / nb_eval_steps,
eval_accuracy / nb_eval_examples))
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
logger.info(f"epoch, step | {epoch_index}, {step}")
logger.info(" | Training | Validation")
logger.info("accuracy | %.4f" %
(acc / nb_tr_examples) +
" | %.4f" % eval_accuracy)
logger.info("loss | %.4f" %
(tr_loss / nb_tr_steps) +
" | %.4f" % eval_loss)
best_acc = max(best_acc, eval_accuracy)
if eval_accuracy == best_acc:
best_epoch = (epoch_index, step)
logger.info(
"best validation performance so far %.4f: " %
best_acc + ", best epoch: " + str(best_epoch) +
". saving current model to " + args.output_dir)
# Save a trained model, configuration and tokenizer
model_to_save = model.module if hasattr(
model,
'module') else model # Only save the model it-self
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(
args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(
args.output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(),
output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
model.train()
epoch_end_time = time.time()
logger.info(
f"time it took to finish the epoch {epoch_index} of {args.num_train_epochs} is "
+ _show_runtime(epoch_end_time - epoch_start_time))
# Does this even make sense to output?
result = {
'eval_accuracy': best_acc,
'global_step': global_step,
'best_epoch': best_epoch
}
cleanup_global_logging(stdout_handler)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
def LoadDataset(args, config, task_cfg, task_id, split="trainval"):
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
else:
tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
task = "TASK" + task_id
task_name = task_cfg[task]["name"]
# initialize the feature reader
feats_h5path1 = task_cfg[task]["features_h5path1"]
feats_h5path2 = task_cfg[task]["features_h5path2"]
features_reader1 = ImageFeaturesH5Reader(feats_h5path1, config, args.in_memory) if feats_h5path1 != "" else None
features_reader2 = ImageFeaturesH5Reader(feats_h5path2, config, args.in_memory) if feats_h5path2 != "" else None
batch_size = task_cfg[task]["batch_size"] // args.grad_acc_steps
num_workers = args.num_workers
if args.local_rank != -1:
batch_size = int(batch_size / dist.get_world_size())
num_workers = int(num_workers / dist.get_world_size())
logger.info("Loading %s Dataset with batch size %d" % (task_name, batch_size))
dset_train, dset_train, task2num_iters = None, None, {}
if "train" in split:
dset_train = DatasetMapTrain[task_name](
task=task_cfg[task]["name"],
dataroot=task_cfg[task]["dataroot"],
annotations_jsonpath=task_cfg[task]["train_annotations_jsonpath"],
split=task_cfg[task]["train_split"],
image_features_reader=features_reader1,
gt_image_features_reader=features_reader2,
tokenizer=tokenizer,
bert_model=args.bert_model,
padding_index=0,
max_seq_length=task_cfg[task]["max_seq_length"],
max_region_num=task_cfg[task]["max_region_num"],
num_locs=config.num_locs,
add_global_imgfeat=config.add_global_imgfeat,
append_mask_sep=(config.fusion_method == 'vl-bert_vqa'),
)
if args.local_rank == -1:
train_sampler = RandomSampler(dset_train)
else:
train_sampler = DistributedSampler(dset_train)
dl_train = DataLoader(
dset_train,
sampler=train_sampler,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=True,
drop_last=args.drop_last,
)
task2num_iters = {task: len(dl_train)}
dset_val, dl_val = None, None
if "val" in split:
dset_val = DatasetMapTrain[task_name](
task=task_cfg[task]["name"],
dataroot=task_cfg[task]["dataroot"],
annotations_jsonpath=task_cfg[task]["val_annotations_jsonpath"],
split=task_cfg[task]["val_split"],
image_features_reader=features_reader1,
gt_image_features_reader=features_reader2,
tokenizer=tokenizer,
bert_model=args.bert_model,
padding_index=0,
max_seq_length=task_cfg[task]["max_seq_length"],
max_region_num=task_cfg[task]["max_region_num"],
num_locs=config.num_locs,
add_global_imgfeat=config.add_global_imgfeat,
append_mask_sep=(config.fusion_method == 'vl-bert_vqa'),
)
dl_val = DataLoader(
dset_val,
shuffle=False,
batch_size=batch_size,
num_workers=2,
pin_memory=True,
drop_last=args.drop_last,
)
return batch_size, task2num_iters, dset_train, dset_val, dl_train, dl_val
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--input_data_path",
default=None,
type=str,
required=True,
help="The training data path")
parser.add_argument("--output_data_path",
default=None,
type=str,
required=True,
help="The validation data path")
parser.add_argument(
"--mcq_model",
default=None,
type=str,
required=True,
help="choose one from the list: bert-mcq-parallel-max, "
"bert-mcq_parallel-weighted-sum, bert-mcq-concat, mac-bert")
parser.add_argument(
"--bert_model",
default=None,
type=str,
required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument(
"--model_dir",
default=None,
type=str,
required=True,
help="The output directory where the model checkpoints will be written."
)
## Other parameters
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help=
"The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--error_only",
action='store_true',
help="Whether to filter errors. Labels are needed")
parser.add_argument(
"--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--eval_batch_size",
default=16,
type=int,
help="Total batch size for eval.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
'--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--max_number_premises',
type=int,
default=None,
help="Number of premise sentences to use at max")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument(
'--tie_weights_weighted_sum',
action='store_true',
help="Whether to tie the weights for the weighted sum model")
parser.add_argument('--with_score',
action='store_true',
help="Knowledge with score is provided")
parser.add_argument('--stamp_weights',
action='store_true',
help="Ignores premises with weights less than 0.1")
args = parser.parse_args()
device = torch.device(
"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
if not os.path.exists(args.model_dir) and not os.listdir(args.model_dir):
raise ValueError("Model directory ({}) doesnot exists.".format(
args.model_dir))
stdout_handler = prepare_global_logging(args.model_dir, False)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if "roberta" in args.bert_model:
tokenizer = RobertaTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case)
logger.info("Type of Tokenizer : ROBERTA")
else:
tokenizer = BertTokenizer.from_pretrained(
args.bert_model, do_lower_case=args.do_lower_case)
logger.info("Type of Tokenizer : BERT")
data_reader = None
if args.mcq_model == 'bert-mcq-parallel-max':
model = BertMCQParallel.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-concat':
model = BertMCQConcat.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQConcatReader()
elif args.mcq_model == 'bert-mcq-weighted-sum':
model = BertMCQWeightedSum.from_pretrained(
args.model_dir,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-simple-sum':
model = BertMCQSimpleSum.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'bert-mcq-mac':
model = BertMCQMAC.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = BertMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-parallel-max':
model = RoBertaMCQParallel.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-concat':
model = RoBertaMCQConcat.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQConcatReader()
elif args.mcq_model == 'roberta-mcq-weighted-sum':
model = RoBertaMCQWeightedSum.from_pretrained(
args.model_dir,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-ws-score':
model = RoBertaMCQWeightedSumScore.from_pretrained(
args.model_dir,
tie_weights=args.tie_weights_weighted_sum,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelScoreReader()
elif args.mcq_model == 'roberta-mcq-simple-sum':
model = RoBertaMCQSimpleSum.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
elif args.mcq_model == 'roberta-mcq-ss-score':
model = RoBertaMCQSimpleSumScore.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelScoreReader()
elif args.mcq_model == 'roberta-mcq-mac':
model = RoBertaMCQMAC.from_pretrained(
args.model_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(args.local_rank)))
data_reader = RoBertaMCQParallelReader()
else:
logger.error(f"Invalid MCQ model name {args.mcq_model}")
exit(0)
# Load Data To Score:
eval_data = data_reader.read(args.input_data_path, tokenizer,
args.max_seq_length, args.max_number_premises)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training."
)
model = DDP(model)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
logger.info("***** Evaluation *****")
logger.info(" num examples = %d", len(eval_data))
logger.info(" batch size = %d", args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
etq = tqdm(eval_dataloader, desc="Scoring")
prediction_list = []
gold_labels = []
scores = []
for input_ids, segment_ids, input_mask, label_ids in etq:
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
outputs = model(input_ids, segment_ids, input_mask, label_ids)
tmp_eval_loss = outputs[0]
logits = outputs[1]
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy, predictions = accuracy(logits, label_ids)
scores.extend(logits)
gold_labels.extend(label_ids)
prediction_list.extend(predictions)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
etq.set_description(
_get_loss_accuracy(eval_loss / nb_eval_steps,
eval_accuracy / nb_eval_examples))
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
cleanup_global_logging(stdout_handler)
output_score_file = os.path.join(args.output_data_path, "score_file.txt")
output_only_preds = os.path.join(args.output_data_path, "predictions.txt")
output_with_labels = os.path.join(args.output_data_path, "pred_labels.txt")
with open(output_score_file, "w") as scorefile:
for score in scores:
scorefile.write(str(softmax(score)) + "\n")
with open(output_only_preds,
"w") as onlypreds, open(output_with_labels, "w") as predlabels:
for pred, label in zip(prediction_list, gold_labels):
onlypreds.write(str(pred) + "\n")
predlabels.write(
str(pred) + "\t" + str(label) + "\t" + str(pred == label) +
"\n")
def get_tokenizer(self):
return RobertaTokenizer.from_pretrained(self.tmpdirname,
**self.special_tokens_map)
