def create_extra_data(task, prev_task, model, train_extra_data):
if args.real_sample:
logger.info(f"using real data as extra data")
return get_real_data(task, train_extra_data)
task_cnt = args.tasks.index(task)
model_dir = get_model_dir([prev_task])
gen_path = os.path.join(model_dir, "lm.csv")
if os.path.exists(gen_path):
logger.info(f"extra data exists in {gen_path}, read it!")
return read_extra_data(gen_path, train_extra_data)
gen_size = DATA_ATTRS[task]["train"]["data_size"]
gen_size = int(np.ceil(gen_size * args.gen_lm_sample_percentage))
gen_size -= (gen_size % task_cnt)
if args.debug:
gen_size = task_cnt
model.eval()
need_process = OrderedDict()
qa_results = []
for task_name in args.tasks[:task_cnt]:
qa_results.extend([
torch.tensor([SPECIAL_TOKEN_IDS[task_name]])
for _ in range(gen_size // task_cnt)
])
all_pasts = [[
torch.empty(2,
MODEL_CONFIG.n_head,
0,
MODEL_CONFIG.n_embd // MODEL_CONFIG.n_head,
dtype=torch.float if args.fp32 else torch.half).cuda()
for _ in range(gen_size)
] for __ in range(MODEL_CONFIG.n_layer)]
max_tot_lens = [args.max_len for _ in range(gen_size)]
for i in range(gen_size):
need_process.update([[i, None]])
if len(need_process) > int(args.memory_sizes[0] * 0.12):
sample_sequence(model, need_process, qa_results, all_pasts,
max_tot_lens)
sample_sequence(model, need_process, qa_results, all_pasts, max_tot_lens)
model.train()
qa_results = [res.tolist() for res in qa_results]
train_extra_data.extend(qa_results)
qa_results = [TOKENIZER.decode(res) for res in qa_results]
write_extra_data(gen_path, qa_results)
def test_one_to_one(task_load, task_eval, model, score_dict):
logger.info("start to test { task: %s (load) %s (eval), seq train type: %s }" % (task_load, task_eval, args.seq_train_type))
test_qadata = QADataset(TASK_DICT[task_eval]["test"] , "test", SPECIAL_TOKEN_IDS[task_load]).sort()
max_a_len = test_qadata.max_a_len
test_dataloader = create_dataloader(test_qadata, "test")
n_examples = len(test_qadata)
logger.info("len of test dataset: {}".format(n_examples))
need_process = OrderedDict()
qa_results = [0 for _ in range(n_examples)]
all_pasts = [[0 for _ in range(n_examples)] for __ in range(MODEL_CONFIG.n_layer)]
max_tot_lens = [0 for _ in range(n_examples)]
cnt = 0
for n_steps, (cqs, len_cqs, _, _, _, _, _) in enumerate(test_dataloader):
# assume n_gpus == 1
cqs = cqs[0]
len_cqs = len_cqs[0]
n_inputs = cqs.shape[0]
all_outputs = model(input_ids=cqs.cuda())
outputs = all_outputs[0]
if args.model_name == "gpt2":
pasts = all_outputs[1]
next_logits = outputs[range(n_inputs), len_cqs-1, :] / args.temperature_qa
next_tokens = logits_to_tokens(next_logits).cpu()
for i in range(n_inputs):
max_tot_lens[cnt] = max_a_len + test_qadata[cnt][1]
qa_results[cnt] = cqs[i][:len_cqs[i]]
if next_tokens[i] != SPECIAL_TOKEN_IDS["eos_token"]:
qa_results[cnt] = torch.cat((cqs[i][:len_cqs[i]], next_tokens[i]))
if len(qa_results[cnt]) not in [max_tot_lens[cnt], args.max_len]:
need_process.update([[cnt, None]])
if args.model_name == "gpt2":
for layer_id in range(MODEL_CONFIG.n_layer):
all_pasts[layer_id][cnt] = pasts[layer_id][:, i, ..., :len_cqs[i], :].type(torch.float32 if args.fp32 else torch.half)
cnt += 1
if len(need_process) > int(12 * args.memory_sizes[0] / cqs.shape[1]): # dynamic threshold to avoid out of memory
sample_sequence(model, need_process, qa_results, all_pasts, max_tot_lens)
sample_sequence(model, need_process, qa_results, all_pasts, max_tot_lens)
if task_eval in ['wikisql','woz.en','multinli.in.out']:
ids = test_qadata.get_indices()
test_qadata.sort_by_index()
qa_results = [x[1] for x in sorted([(i, g) for i, g in zip(ids, qa_results)])]
for i in range(len(test_qadata)):
_, len_cq, _, _, Y, _, _, _ = test_qadata[i]
if task_eval in ['wikisql','woz.en']:
Y = test_qadata.answers[i]
else:
Y = list(filter(lambda x: x != -1, Y))[:-1] # remove eos
Y = ' '.join([str(y) for y in Y]).split(str(SPECIAL_TOKEN_IDS["pad_token"]))
Y = [TOKENIZER.decode(list(map(int, y.split()))) for y in Y]
qa_results[i] = [TOKENIZER.decode(qa_results[i].tolist()[len_cq:]), Y]
get_test_score(task_eval, qa_results, score_dict)
model_dir = model.model_dir
ep = model.ep
results_path = os.path.join(model_dir,"qa_{}_{}.csv".format(task_eval,ep+1))
if not args.debug:
with open(results_path, "w",encoding="utf-8") as f:
qa_writer = csv.writer(f,delimiter=',')
qa_writer.writerow(["y","pred"])
for pred, y in qa_results:
if task_eval == 'wikisql':
y = y["answer"]
elif task_eval == 'woz.en':
y = y[1]
qa_writer.writerow([y,pred])
return model, score_dict