def hotpotqa_preprocess_example():
start_time = time()
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=PRE_TAINED_LONFORMER_BASE)
longformer_tokenizer = LongformerQATensorizer(tokenizer=tokenizer,
max_length=-1)
dev_data, _ = HOTPOT_DevData_Distractor()
print('*' * 75)
dev_test_data = Hotpot_Test_Data_PreProcess(data=dev_data,
tokenizer=longformer_tokenizer)
print('Get {} dev-test records'.format(dev_test_data.shape[0]))
dev_test_data.to_json(
os.path.join(abs_distractor_wiki_path,
'hotpot_test_distractor_wiki_tokenized.json'))
print('*' * 75)
dev_data, _ = HOTPOT_DevData_Distractor()
dev_data = Hotpot_Dev_Data_Preprocess(data=dev_data,
tokenizer=longformer_tokenizer)
print('Get {} dev records'.format(dev_data.shape[0]))
dev_data.to_json(
os.path.join(abs_distractor_wiki_path,
'hotpot_dev_distractor_wiki_tokenized.json'))
print('*' * 75)
train_data, _ = HOTPOT_TrainData()
train_data = Hotpot_Train_Data_Preprocess(data=train_data,
tokenizer=longformer_tokenizer)
print('Get {} training records'.format(train_data.shape[0]))
train_data.to_json(
os.path.join(abs_distractor_wiki_path,
'hotpot_train_distractor_wiki_tokenized.json'))
print('Runtime = {:.4f} seconds'.format(time() - start_time))
print('*' * 75)
def test_data_loader_checker():
file_path = '../data/hotpotqa/distractor_qa'
dev_file_name = 'hotpot_dev_distractor_wiki_tokenized.json'
from torch.utils.data import DataLoader
batch_size = 1
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
longtokenizer = get_hotpotqa_longformer_tokenizer()
hotpot_tensorizer = LongformerQATensorizer(tokenizer=longtokenizer,
max_length=4096)
start_time = time()
test_dataloader = DataLoader(HotpotDevDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=1,
collate_fn=HotpotDevDataset.collate_fn)
for batch_idx, sample in enumerate(test_dataloader):
sd_mask = sample['sd_mask']
# print(sd_mask)
# print(sd_mask[0])
print(sample['doc_lens'])
print(sample['sent_lens'])
ss_mask = sample['ss_mask']
# print(ss_mask[0].detach().tolist())
print(ss_mask.shape)
print(ss_mask[0].sum(dim=1))
print(sd_mask.shape)
break
print('Runtime = {}'.format(time() - start_time))
def get_model(args):
start_time = time()
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=args.pretrained_cfg_name)
longEncoder = LongformerEncoder.init_encoder(
cfg_name=args.pretrained_cfg_name,
projection_dim=args.project_dim,
hidden_dropout=args.input_drop,
attn_dropout=args.attn_drop,
seq_project=args.seq_project)
longEncoder.resize_token_embeddings(len(tokenizer))
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if args.frozen_layer_num > 0:
modules = [
longEncoder.embeddings,
*longEncoder.encoder.layer[:args.frozen_layer_num]
]
for module in modules:
for param in module.parameters():
param.requires_grad = False
logging.info('Frozen the first {} layers'.format(
args.frozen_layer_num))
logging.info('Loading encoder takes {:.4f}'.format(time() - start_time))
model = LongformerHotPotQAModel(longformer=longEncoder,
num_labels=args.num_labels)
logging.info(
'Constructing reasonModel completes in {:.4f}'.format(time() -
start_time))
return model
def data_loader_checker():
file_path = '../data/hotpotqa/distractor_qa'
dev_file_name = 'hotpot_dev_distractor_wiki_tokenized.json'
from torch.utils.data import DataLoader
batch_size = 6
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
for col in data_frame.columns:
print(col)
longtokenizer = get_hotpotqa_longformer_tokenizer()
hotpot_tensorizer = LongformerQATensorizer(tokenizer=longtokenizer,
max_length=4096)
start_time = time()
dev_dataloader = DataLoader(HotpotDevDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=6,
collate_fn=HotpotDevDataset.collate_fn)
for batch_idx, sample in enumerate(dev_dataloader):
x = sample['doc_start']
# print(sample['doc_start'].shape)
# print(sample['sent_start'].shape)
print('Runtime = {}'.format(time() - start_time))
def performance_collection(folder_name):
print('Loading tokenizer')
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=PRE_TAINED_LONFORMER_BASE, do_lower_case=True)
json_file_names = get_all_json_files(file_path=folder_name)
json_file_names = [x for x in json_file_names if x != 'config.json']
print('{} json files have been found'.format(len(json_file_names)))
max_sp_sent_f1 = 0
max_metric_res = None
max_json_file_name = None
for idx, json_file_name in enumerate(json_file_names):
if json_file_name != 'config.json':
data_frame_i = load_data_frame_align_with_dev(
file_path=folder_name, json_fileName=json_file_name)
metrics_i = convert2leadBoard(data=data_frame_i,
tokenizer=tokenizer)
if max_sp_sent_f1 < metrics_i['sp_f1']:
max_sp_sent_f1 = metrics_i['sp_f1']
max_metric_res = metrics_i
max_json_file_name = json_file_name
print_metrics(name=json_file_name, metrics=metrics_i)
print('*' * 75)
print('+' * 75)
print_metrics(name=max_json_file_name, metrics=max_metric_res)
print('+' * 75)
def get_model(args):
start_time = time()
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=args.pretrained_cfg_name)
longEncoder = LongformerEncoder.init_encoder(
cfg_name=args.pretrained_cfg_name,
projection_dim=args.project_dim,
hidden_dropout=args.input_drop,
attn_dropout=args.attn_drop,
seq_project=args.seq_project)
longEncoder.resize_token_embeddings(len(tokenizer))
model = LongformerHotPotQAModel(longformer=longEncoder,
num_labels=args.num_labels,
args=args)
logging.info(
'Constructing reasonModel completes in {:.4f}'.format(time() -
start_time))
return model
def get_test_data_loader(args):
data_frame = read_train_dev_data_frame(file_path=args.data_path,
json_fileName=args.dev_data_name)
batch_size = args.test_batch_size
data_size = data_frame.shape[0]
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=args.pretrained_cfg_name, do_lower_case=True)
hotpot_tensorizer = LongformerQATensorizer(tokenizer=tokenizer,
max_length=args.max_ctx_len)
dataloader = DataLoader(HotpotDevDataset(
data_frame=data_frame,
hotpot_tensorizer=hotpot_tensorizer,
max_sent_num=args.max_sent_num,
global_mask_type=args.global_mask_type),
batch_size=batch_size,
shuffle=False,
num_workers=max(1, args.cpu_num // 2),
collate_fn=HotpotDevDataset.collate_fn)
return dataloader, data_size
def get_train_data_loader(args):
data_frame = read_train_dev_data_frame(file_path=args.data_path,
json_fileName=args.train_data_name)
batch_size = args.batch_size
#####################################################
training_data_shuffle = args.training_shuffle == 1
#####################################################
data_size = data_frame.shape[0]
if args.train_data_filtered == 1:
data_frame = data_frame[data_frame['level'] != 'easy']
logging.info('Filtered data by removing easy case {} to {}'.format(
data_size, data_frame.shape[0]))
elif args.train_data_filtered == 2:
data_frame = data_frame[data_frame['level'] == 'hard']
logging.info(
'Filtered data by removing easy and medium case {} to {}'.format(
data_size, data_frame.shape[0]))
else:
logging.info('Using all training data {}'.format(data_size))
data_size = data_frame.shape[0]
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=args.pretrained_cfg_name, do_lower_case=True)
hotpot_tensorizer = LongformerQATensorizer(tokenizer=tokenizer,
max_length=args.max_ctx_len)
dataloader = DataLoader(HotpotTrainDataset(
data_frame=data_frame,
hotpot_tensorizer=hotpot_tensorizer,
max_doc_num=args.max_doc_num,
max_sent_num=args.max_sent_num,
global_mask_type=args.global_mask_type,
training_shuffle=training_data_shuffle),
batch_size=batch_size,
shuffle=True,
num_workers=max(1, args.cpu_num // 2),
collate_fn=HotpotTrainDataset.collate_fn)
return dataloader, data_size
def main(model_args):
args = get_config(PATH=model_args.model_path,
config_json_name=model_args.model_config_name)
args.check_point = model_args.model_name
args.data_path = model_args.data_path
args.test_batch_size = model_args.test_batch_size
args.doc_threshold = model_args.doc_threshold
args.save_path = model_args.model_path
if torch.cuda.is_available():
args.cuda = True
else:
args.cuda = False
###################
if args.data_path is None:
raise ValueError('one of data_path must be chosed.')
if args.save_path and not os.path.exists(args.save_path):
os.makedirs(args.save_path)
set_logger(args)
########+++++++++++++++++++++++++++++
abs_path = os.path.abspath(args.data_path)
args.data_path = abs_path
########+++++++++++++++++++++++++++++
# Write logs to checkpoint and console
if args.cuda:
if args.gpu_num > 1:
device_ids, used_memory = gpu_setting(args.gpu_num)
else:
device_ids, used_memory = gpu_setting()
if used_memory > 100:
logging.info('Using memory = {}'.format(used_memory))
if device_ids is not None:
if len(device_ids) > args.gpu_num:
device_ids = device_ids[:args.gpu_num]
device = torch.device('cuda:{}'.format(device_ids[0]))
else:
device = torch.device('cuda:0')
logging.info('Set the cuda with idxes = {}'.format(device_ids))
logging.info('cuda setting {}'.format(device))
logging.info('GPU setting')
else:
device_ids = None
device = torch.device('cpu')
logging.info('CPU setting')
########+++++++++++++++++++++++++++++
logging.info('Loading development data...')
test_data_loader, _ = get_test_data_loader(args=args)
logging.info('Loading data completed')
logging.info('*' * 75)
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
logging.info('Loading Model...')
model = get_model(args=args).to(device)
##+++++++++++
model_path = args.save_path
model_file_name = args.check_point
hotpot_qa_model_name = os.path.join(model_path, model_file_name)
model = load_model(model=model, PATH=hotpot_qa_model_name)
model = model.to(device)
if device_ids is not None:
if len(device_ids) > 1:
model = DataParallel(model,
device_ids=device_ids,
output_device=device)
logging.info('Data Parallel model setting')
##+++++++++++
logging.info('Model Parameter Configuration:')
for name, param in model.named_parameters():
logging.info('Parameter {}: {}, require_grad = {}'.format(
name, str(param.size()), str(param.requires_grad)))
logging.info('*' * 75)
logging.info("Model hype-parameter information...")
for key, value in vars(args).items():
logging.info('Hype-parameter\t{} = {}'.format(key, value))
logging.info('*' * 75)
logging.info("Model hype-parameter information...")
for key, value in vars(model_args).items():
logging.info('Hype-parameter\t{} = {}'.format(key, value))
logging.info('*' * 75)
logging.info('projection_dim = {}'.format(args.project_dim))
logging.info('Multi-task encoding')
logging.info('*' * 75)
logging.info('Loading tokenizer')
tokenizer = get_hotpotqa_longformer_tokenizer()
logging.info('*' * 75)
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
# logging.info('Multi-task encoding')
# metric_dict = multi_task_decoder(model=model, device=device, test_data_loader=test_data_loader, args=args)
# answer_type_acc = metric_dict['answer_type_acc']
# logging.info('*' * 75)
# logging.info('Answer type prediction accuracy: {}'.format(answer_type_acc))
# logging.info('*' * 75)
# for key, value in metric_dict.items():
# if key.endswith('metrics'):
# logging.info('{} prediction'.format(key))
# log_metrics('Valid', 'final', value)
# logging.info('*' * 75)
# ##++++++++++++++++++++++++++++++++++++++++++++++++++++
# dev_data_frame = metric_dict['res_dataframe']
# ##################################################
# leadboard_metric, res_data_frame = convert2leadBoard(data=dev_data_frame, tokenizer=tokenizer)
# ##=================================================
# logging.info('*' * 75)
# log_metrics('Evaluation', step='leadboard', metrics=leadboard_metric)
# logging.info('*' * 75)
# date_time_str = get_date_time()
# dev_result_name = os.path.join(args.save_path,
# date_time_str + '_mt_evaluation.json')
# res_data_frame.to_json(dev_result_name, orient='records')
# logging.info('Saving {} record results to {}'.format(res_data_frame.shape, dev_result_name))
# logging.info('*' * 75)
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
logging.info('Hierarchical encoding')
metric_dict = hierartical_decoder(model=model,
device=device,
test_data_loader=test_data_loader,
doc_topk=model_args.doc_topk,
args=args)
answer_type_acc = metric_dict['answer_type_acc']
logging.info('*' * 75)
logging.info('Answer type prediction accuracy: {}'.format(answer_type_acc))
logging.info('*' * 75)
for key, value in metric_dict.items():
if key.endswith('metrics'):
logging.info('{} prediction'.format(key))
log_metrics('Valid', 'final', value)
logging.info('*' * 75)
##++++++++++++++++++++++++++++++++++++++++++++++++++++
topk_dev_data_frame = metric_dict['topk_dataframe']
##################################################
topk_leadboard_metric, topk_res_data_frame = convert2leadBoard(
data=topk_dev_data_frame, tokenizer=tokenizer)
##=================================================
log_metrics('Topk Evaluation',
step='leadboard',
metrics=topk_leadboard_metric)
date_time_str = get_date_time()
topk_dev_result_name = os.path.join(
args.save_path, date_time_str + '_topk_hi_evaluation.json')
topk_res_data_frame.to_json(topk_dev_result_name, orient='records')
logging.info('Saving {} record results to {}'.format(
topk_res_data_frame.shape, topk_dev_result_name))
logging.info('*' * 75)
##=================================================
thresh_dev_data_frame = metric_dict['thresh_dataframe']
##################################################
thresh_leadboard_metric, thresh_res_data_frame = convert2leadBoard(
data=thresh_dev_data_frame, tokenizer=tokenizer)
log_metrics('Thresh Evaluation',
step='leadboard',
metrics=thresh_leadboard_metric)
##=================================================
date_time_str = get_date_time()
thresh_dev_result_name = os.path.join(
args.save_path, date_time_str + '_thresh_hi_evaluation.json')
thresh_res_data_frame.to_json(thresh_dev_result_name, orient='records')
logging.info('Saving {} record results to {}'.format(
thresh_res_data_frame.shape, thresh_dev_result_name))
logging.info('*' * 75)
def answer_consistent_checker():
file_path = '../data/hotpotqa/distractor_qa'
dev_file_name = 'hotpot_dev_distractor_wiki_tokenized.json'
from torch.utils.data import DataLoader
batch_size = 1
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
print(data_frame['answer_len'].max())
# for col in data_frame.columns:
# print(col)
longtokenizer = get_hotpotqa_longformer_tokenizer()
hotpot_tensorizer = LongformerQATensorizer(tokenizer=longtokenizer,
max_length=4096)
start_time = time()
dev_dataloader = DataLoader(HotpotTrainDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=14,
collate_fn=HotpotTrainDataset.collate_fn)
max_seq_len = 0
average_seq_len = 0
count = 0
max_answer_len = 0
for batch_idx, sample in enumerate(dev_dataloader):
# if batch_idx % 1000 == 0:
# print(batch_idx)
ctx_encode = sample['ctx_encode']
ctx_encode_lens = sample['doc_lens']
answer_start = sample['ans_start'].squeeze(dim=-1)
answer_end = sample['ans_end'].squeeze(dim=-1)
doc_start = sample['doc_start'].squeeze(dim=-1)
doc_end = sample['doc_end'].squeeze(dim=-1)
sent_start = sample['sent_start'].squeeze(dim=-1)
batch_size = ctx_encode.shape[0]
for id in range(batch_size):
# doc_token_num = ctx_encode_lens[id].sum().data.item()
doc_token_num = doc_end[id].detach().tolist()[-1]
if max_seq_len < doc_token_num:
max_seq_len = doc_token_num
average_seq_len = average_seq_len + doc_token_num
count = count + 1
doc_start_i = doc_start[id]
sent_start_i = sent_start[id]
ctx_encode_i = ctx_encode[id]
ans_start_i = answer_start[id].data.item()
ans_end_i = answer_end[id].data.item()
if max_answer_len < (ans_end_i - ans_start_i) + 1:
max_answer_len = (ans_end_i - ans_start_i) + 1
decode_answer = longtokenizer.decode(
ctx_encode_i[ans_start_i:(ans_end_i + 1)])
# print('{}\t{}'.format(batch_idx, decode_answer))
# if '<p>' in decode_answer or '<d>' in decode_answer or '<q>' in decode_answer or '</q>' in decode_answer:
# print('index = {}'.format(batch_idx))
# print('decode answer {}'.format(decode_answer))
# print('Decode Query {}'.format(longtokenizer.decode(ctx_encode_i[:doc_start_i[0]])))
# print('decode answer {}'.format(decode_answer))
print('max seq len: {} average seq len: {}, {}'.format(
max_seq_len, average_seq_len / count, count))
print('max answer len: {}'.format(max_answer_len))
return
def data_consistent_checker(train=True):
file_path = '../data/hotpotqa/distractor_qa'
from torch.utils.data import DataLoader
batch_size = 2
longtokenizer = get_hotpotqa_longformer_tokenizer()
hotpot_tensorizer = LongformerQATensorizer(tokenizer=longtokenizer,
max_length=4096)
if train:
dev_file_name = 'hotpot_train_distractor_wiki_tokenized.json'
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
start_time = time()
dev_dataloader = DataLoader(HotpotTrainDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=1,
collate_fn=HotpotTrainDataset.collate_fn)
else:
dev_file_name = 'hotpot_dev_distractor_wiki_tokenized.json'
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
start_time = time()
dev_dataloader = DataLoader(HotpotDevDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=1,
collate_fn=HotpotDevDataset.collate_fn)
batch_data_frame = data_frame.head(batch_size)
print(batch_data_frame.shape)
for idx, row in batch_data_frame.iterrows():
context = row['context']
supp_fact_filtered = row['supp_facts_filtered']
# for supp, sen_idx in supp_fact_filtered:
# print('Support doc: {}, sent id: {}'.format(supp, sen_idx))
print('Query {}'.format(row['question']))
for doc_idx, doc in enumerate(context):
# print('doc {}: title = {} \n text = {}'.format(doc_idx + 1, doc[0], '\n'.join(doc[1])))
print('doc {}: title = {}'.format(doc_idx + 1, doc[0]))
for supp, sen_idx in supp_fact_filtered:
if doc[0] == supp:
print('supp fact doc {}: sent = {} text = {}'.format(
doc_idx, sen_idx, doc[1][sen_idx]))
print('*' * 70)
print('Original answer = {}'.format(row['norm_answer']))
print('=' * 70)
print('+' * 70)
print('\n' * 3)
for batch_idx, sample in enumerate(dev_dataloader):
# for key, value in sample.items():
# print(key)
ctx_encode = sample['ctx_encode']
ctx_marker_mask = sample['marker']
global_atten = sample['ctx_global_mask']
atten_mask = sample['ctx_attn_mask']
sup_sent_labels = sample['sent_labels'].squeeze(dim=-1)
sent2doc_map = sample['s2d_map']
sentIndoc_map = sample['sInd_map']
sent_start = sample['sent_start']
sent_end = sample['sent_end']
# print('sent num = {}'.format(sent_end.shape[1]))
answer_start = sample['ans_start'].squeeze(dim=-1)
answer_end = sample['ans_end'].squeeze(dim=-1)
doc_start = sample['doc_start'].squeeze(dim=-1)
token2sent_map = sample['t2s_map'].squeeze(dim=-1)
if train:
head_idx = sample['head_idx'].squeeze(dim=-1)
tail_idx = sample['tail_idx'].squeeze(dim=-1)
for id in range(batch_size):
ctx_marker_i = ctx_marker_mask[id]
supp_idxes = (sup_sent_labels[id] > 0).nonzero().squeeze()
doc_idxes = sent2doc_map[id][supp_idxes].detach().tolist()
sent_idxes = sentIndoc_map[id][supp_idxes].detach().tolist()
doc_start_i = doc_start[id]
doc_sent_pairs = list(zip(doc_idxes, sent_idxes))
sent_start_i = sent_start[id]
sent_end_i = sent_end[id]
ctx_encode_i = ctx_encode[id]
token2sent_map_i = token2sent_map[id]
# print('token to sentence {}'.format(token2sent_map_i.max()))
max_sent_num = token2sent_map_i.max().data.item()
for ssss_id in range(max_sent_num):
sent_iiii_idexs = (
token2sent_map_i == ssss_id).nonzero().squeeze()
print('sent {} text = {}'.format(
ssss_id,
longtokenizer.decode(ctx_encode_i[sent_iiii_idexs])))
if train:
print('head doc idx = {}'.format(head_idx[id]))
print('tail doc idx = {}'.format(tail_idx[id]))
global_atten_i = global_atten[id]
global_atten_i_indexes = (global_atten_i > 0).nonzero().squeeze()
global_atten_text = longtokenizer.decode(
ctx_encode_i[global_atten_i_indexes])
print('global attention text: {}'.format(global_atten_text))
atten_i = atten_mask[id]
atten_i_indexes = (atten_i > 0).nonzero().squeeze()
atten_text = longtokenizer.decode(ctx_encode_i[atten_i_indexes])
# print('attention text: {}'.format(atten_text))
print('x' * 75)
# print('decode text: {}'.format(longtokenizer.decode(ctx_encode_i)))
ans_start_i = answer_start[id].data.item()
ans_end_i = answer_end[id].data.item()
#
print('Decode Query {}'.format(
longtokenizer.decode(ctx_encode_i[:doc_start_i[0]])))
print('Decode Answer {}'.format(
longtokenizer.decode(ctx_encode_i[ans_start_i:(ans_end_i +
1)])))
ctx_marker_i_indexes = (ctx_marker_i > 0).nonzero().squeeze()
print('Decode marker text = {}'.format(
longtokenizer.decode(ctx_encode_i[ctx_marker_i_indexes])))
for ss_id, x in enumerate(doc_sent_pairs):
supp_idddd = supp_idxes[ss_id]
start_i, end_i = sent_start_i[
supp_idddd], sent_end_i[supp_idddd] + 1
print('doc {}, sent {}, text {}'.format(
x[0], x[1],
longtokenizer.decode(ctx_encode_i[start_i:end_i])))
print('=' * 70)
break
return
def data_loader_consistent_checker(train=True):
file_path = '../data/hotpotqa/distractor_qa'
if train:
dev_file_name = 'hotpot_train_distractor_wiki_tokenized.json'
else:
dev_file_name = 'hotpot_dev_distractor_wiki_tokenized.json'
data_frame = read_train_dev_data_frame(PATH=file_path,
json_fileName=dev_file_name)
longtokenizer = get_hotpotqa_longformer_tokenizer()
hotpot_tensorizer = LongformerQATensorizer(tokenizer=longtokenizer,
max_length=4096)
start_time = time()
from torch.utils.data import DataLoader
batch_size = 1
if train:
dev_dataloader = DataLoader(HotpotTrainDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=1,
collate_fn=HotpotTrainDataset.collate_fn)
else:
dev_dataloader = DataLoader(HotpotDevDataset(
data_frame=data_frame, hotpot_tensorizer=hotpot_tensorizer),
batch_size=batch_size,
shuffle=False,
num_workers=1,
collate_fn=HotpotDevDataset.collate_fn)
head_two = data_frame.head(batch_size)
print(type(head_two))
for idx, row in head_two.iterrows():
context = row['context']
supp_fact_filtered = row['supp_facts_filtered']
for supp, sen_idx in supp_fact_filtered:
print('Support doc: {}, sent id: {}'.format(supp, sen_idx))
print('-' * 70)
print()
print('Query {}'.format(row['question']))
for doc_idx, doc in enumerate(context):
print('doc {}: title = {} \n text = {}'.format(
doc_idx + 1, doc[0], ' '.join(doc[1])))
print('-' * 70)
print('*' * 70)
print()
print('Original answer = {}'.format(row['norm_answer']))
print('=' * 70)
print('+' * 70)
print('\n' * 5)
for batch_idx, sample in enumerate(dev_dataloader):
ctx_encode = sample['ctx_encode']
doc_start = sample['doc_start'].squeeze(dim=-1)
sent_start = sample['sent_start'].squeeze(dim=-1)
answer_start = sample['ans_start'].squeeze(dim=-1)
answer_end = sample['ans_end'].squeeze(dim=-1)
if train:
head_idx = sample['head_idx'].squeeze(dim=-1)
tail_idx = sample['tail_idx'].squeeze(dim=-1)
sent_lens = sample['sent_lens'].squeeze(dim=-1)
attention = sample['ctx_attn_mask'].squeeze(dim=-1)
global_attenion = sample['ctx_global_mask']
print('global attention {}'.format(global_attenion))
marker = sample['marker'].squeeze(dim=-1)
doc_num = doc_start.shape[1]
print('doc num: {}'.format(doc_start.shape))
print('marker {}'.format(marker))
print('marker shape {}'.format(marker.shape))
for idx in range(ctx_encode.shape[0]):
ctx_i = ctx_encode[idx]
marker_i = marker[idx]
marker_idx = marker_i.nonzero().squeeze()
print('marker text {}'.format(
longtokenizer.decode(ctx_i[marker_idx])))
print('*' * 75)
attention_i = attention[idx]
attn_idx = (attention_i == 1).nonzero().squeeze()
print('attn text {}'.format(longtokenizer.decode(ctx_i[attn_idx])))
sent_start_i = sent_start[idx]
doc_start_i = doc_start[idx]
if train:
head_i = head_idx[idx].data.item()
tail_i = tail_idx[idx].data.item()
ans_start_i = answer_start[idx].data.item()
ans_end_i = answer_end[idx].data.item()
print('Decode Query {}'.format(
longtokenizer.decode(ctx_i[:doc_start_i[0]])))
print('*' * 75)
print('Decoded answer = {}'.format(
hotpot_tensorizer.to_string(ctx_i[ans_start_i:(ans_end_i +
1)])))
print('*' * 75)
# print(ans_start_i)
doc_marker = longtokenizer.decode(ctx_i[doc_start_i])
print('doc_marker: {}'.format(doc_marker))
sent_marker = longtokenizer.decode(ctx_i[sent_start_i])
print('doc: {}\nsent: {}\n{}\n{}'.format(doc_marker, sent_marker,
sent_start_i.shape,
sent_lens[idx]))
print('*' * 75)
for k in range(doc_num):
if k < doc_num - 1:
# doc_k = hotpot_tensorizer.to_string(ctx_i[doc_start_i[k]:doc_start_i[k+1]])
doc_k = longtokenizer.decode(
ctx_i[doc_start_i[k]:doc_start_i[k + 1]])
else:
# doc_k = hotpot_tensorizer.to_string(ctx_i[doc_start_i[k]:])
doc_k = longtokenizer.decode(ctx_i[doc_start_i[k]:])
# print(doc_marker)
print('Supp doc {}: text = {}'.format(k + 1, doc_k))
if train:
if k == head_i:
print('=' * 70)
print('Head positive doc {}: text: {}'.format(
head_i + 1, doc_k))
print('=' * 70)
if k == tail_i:
print('=' * 70)
print('Tail positive doc {}: text: {}'.format(
tail_i + 1, doc_k))
print('=' * 70)
print('-' * 70)
print('*' * 70)
print()
# print(ctx_encode.shape)
break
print('Runtime = {}'.format(time() - start_time))
def main(args):
set_seeds(args.rand_seed)
if (not args.do_train) and (not args.do_valid) and (not args.do_test):
raise ValueError('one of train/val/test mode must be chosed.')
if args.data_path is None:
raise ValueError('one of init_checkpoint/data_path must be chosed.')
if args.do_train and args.save_path is None:
raise ValueError('Where do you want to save your trained reasonModel?')
if args.save_path and not os.path.exists(args.save_path):
os.makedirs(args.save_path)
########+++++++++++++++++++++++++++++
abs_path = os.path.abspath(args.data_path)
args.data_path = abs_path
########+++++++++++++++++++++++++++++
# Write logs to checkpoint and console
set_logger(args)
if args.cuda:
if args.do_debug:
if args.gpu_num > 1:
device_ids, used_memory = gpu_setting(args.gpu_num)
else:
device_ids, used_memory = gpu_setting()
if used_memory > 100:
logging.info('Using memory = {}'.format(used_memory))
if device_ids is not None:
if len(device_ids) > args.gpu_num:
device_ids = device_ids[:args.gpu_num]
device = torch.device('cuda:{}'.format(device_ids[0]))
else:
device = torch.device('cuda:0')
logging.info('Set the cuda with idxes = {}'.format(device_ids))
logging.info('cuda setting {}'.format(device))
else:
if args.gpu_num > 1:
logging.info("Using GPU!")
available_device_count = torch.cuda.device_count()
logging.info('GPU number is {}'.format(available_device_count))
if args.gpu_num > available_device_count:
args.gpu_num = available_device_count
# ++++++++++++++++++++++++++++++++++
device_ids, used_memory = gpu_setting(args.gpu_num)
# ++++++++++++++++++++++++++++++++++
device = torch.device("cuda:{}".format(device_ids[0]))
# ++++++++++++++++++++++++++++++++++
else:
device = torch.device("cuda:0")
device_ids = None
logging.info('Single GPU setting')
else:
device = torch.device('cpu')
device_ids = None
logging.info('CPU setting')
logging.info('Device = {}, Device ids = {}'.format(device, device_ids))
logging.info('Loading training data...')
train_data_loader, train_data_size = get_train_data_loader(args=args)
estimated_max_steps = args.epoch * (
(train_data_size // args.batch_size) + 1)
if estimated_max_steps > args.max_steps:
args.max_steps = args.epoch * (
(train_data_size // args.batch_size) + 1)
logging.info('Loading development data...')
dev_data_loader, _ = get_dev_data_loader(args=args)
logging.info('Loading data completed')
logging.info('*' * 75)
tokenizer = get_hotpotqa_longformer_tokenizer()
logging.info('*' * 75)
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if args.do_train:
# Set training configuration
start_time = time()
logging.info('Loading Model...')
model = get_model(args=args).to(device)
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay)
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if device_ids is not None:
if len(device_ids) > 1:
model = DataParallel(model,
device_ids=device_ids,
output_device=device)
logging.info('Data Parallel model setting')
logging.info('Model Parameter Configuration:')
for name, param in model.named_parameters():
logging.info('Parameter {}: {}, require_grad = {}'.format(
name, str(param.size()), str(param.requires_grad)))
logging.info('*' * 75)
logging.info("Model hype-parameter information...")
for key, value in vars(args).items():
logging.info('Hype-parameter\t{} = {}'.format(key, value))
logging.info('*' * 75)
logging.info('batch_size = {}'.format(args.batch_size))
logging.info('projection_dim = {}'.format(args.project_dim))
logging.info('learning_rate = {}'.format(args.learning_rate))
logging.info('Start training...')
train_all_steps(model=model,
optimizer=optimizer,
dev_dataloader=dev_data_loader,
device=device,
train_dataloader=train_data_loader,
tokenizer=tokenizer,
args=args)
logging.info('Completed training in {:.4f} seconds'.format(time() -
start_time))
logging.info('Evaluating on Valid Dataset...')
metric_dict = test_all_steps(model=model,
tokenizer=tokenizer,
test_data_loader=dev_data_loader,
args=args)
answer_type_acc = metric_dict['answer_type_acc']
logging.info('*' * 75)
logging.info(
'Answer type prediction accuracy: {}'.format(answer_type_acc))
logging.info('*' * 75)
for key, value in metric_dict.items():
if key.endswith('metrics'):
logging.info('{} prediction'.format(key))
log_metrics('Valid', 'final', value)
logging.info('*' * 75)
##++++++++++++++++++++++++++++++++++++++++++++++++++++
##++++++++++++++++++++++++++++++++++++++++++++++++++++
model_save_path = save_check_point(model=model,
optimizer=optimizer,
step='all_step',
loss=None,
eval_metric=None,
args=args)
logging.info('Saving the mode in {}'.format(model_save_path))
def hotpot_test_prediction(model, test_data_loader, args):
model.eval()
###########################################################
start_time = time()
step = 0
N = 0
total_steps = len(test_data_loader)
# **********************************************************
answer_type_predicted = []
answer_span_predicted = []
supp_sent_predicted = []
supp_doc_predicted = []
# **********************************************************
with torch.no_grad():
for test_sample in test_data_loader:
if args.cuda:
sample = dict()
for key, value in test_sample.items():
sample[key] = value.cuda()
else:
sample = test_sample
output = model(sample)
N = N + sample['ctx_encode'].shape[0]
# ++++++++++++++++++
answer_type_res = output['yn_score']
if len(answer_type_res.shape) > 1:
answer_type_res = answer_type_res.squeeze(dim=-1)
answer_types = torch.argmax(answer_type_res, dim=-1)
answer_type_predicted += answer_types.detach().tolist()
# +++++++++++++++++++
start_logits, end_logits = output['span_score']
predicted_span_start = torch.argmax(start_logits, dim=-1)
predicted_span_end = torch.argmax(end_logits, dim=-1)
predicted_span_start = predicted_span_start.detach().tolist()
predicted_span_end = predicted_span_end.detach().tolist()
predicted_span_pair = list(
zip(predicted_span_start, predicted_span_end))
answer_span_predicted += predicted_span_pair
# ++++++++++++++++++
supp_doc_res = output['doc_score']
doc_lens = sample['doc_lens']
doc_mask = doc_lens.masked_fill(doc_lens > 0, 1)
supp_doc_pred_i = supp_doc_prediction(scores=supp_doc_res,
mask=doc_mask,
pred_num=2)
supp_doc_predicted += supp_doc_pred_i
# ++++++++++++++++++
supp_sent_res = output['sent_score']
sent_lens = sample['sent_lens']
sent_mask = sent_lens.masked_fill(sent_lens > 0, 1)
sent_fact_doc_idx, sent_fact_sent_idx = sample['fact_doc'], sample[
'fact_sent']
supp_sent_pred_i = supp_sent_prediction(
scores=supp_sent_res,
mask=sent_mask,
doc_fact=sent_fact_doc_idx,
sent_fact=sent_fact_sent_idx,
pred_num=2,
threshold=args.sent_threshold)
supp_sent_predicted += supp_sent_pred_i
# +++++++++++++++++++
step += 1
if step % args.test_log_steps == 0:
logging.info(
'Testing the reasonModel... {}/{} in {:.4f} seconds'.
format(step, total_steps,
time() - start_time))
##=================================================
logging.info('Testing complete...')
logging.info('Loading tokenizer')
tokenizer = get_hotpotqa_longformer_tokenizer(
model_name=PRE_TAINED_LONFORMER_BASE, do_lower_case=True)
logging.info('Loading preprocessed data...')
data = read_train_dev_data_frame(file_path=args.data_path,
json_fileName=args.test_data_name)
data['answer_prediction'] = answer_type_predicted
data['answer_span_prediction'] = answer_span_predicted
data['supp_doc_prediction'] = supp_doc_predicted
data['supp_sent_prediction'] = supp_sent_predicted
def row_process(row):
answer_prediction = row['answer_prediction']
answer_span_predicted = row['answer_span_prediction']
span_start, span_end = answer_span_predicted
encode_ids = row['ctx_encode']
if answer_prediction > 0:
predicted_answer = 'yes' if answer_prediction == 1 else 'no'
else:
predicted_answer = tokenizer.decode(
encode_ids[span_start:(span_end + 1)],
skip_special_tokens=True)
ctx_contents = row['context']
supp_doc_prediction = row['supp_doc_prediction']
supp_doc_titles = [ctx_contents[idx][0] for idx in supp_doc_prediction]
supp_sent_prediction = row['supp_sent_prediction']
supp_sent_pairs = [(ctx_contents[pair_idx[0]][0], pair_idx[1])
for pair_idx in supp_sent_prediction]
return predicted_answer, supp_doc_titles, supp_sent_pairs
res_names = ['answer', 'sp_doc', 'sp']
data[res_names] = data.apply(lambda row: pd.Series(row_process(row)),
axis=1)
return data