def get_best_ckpt(val_data_dir, opts):
pat = re.compile(r'val_results_(?P<step>\d+)_rank0.csv')
prediction_files = glob.glob('{}/results/val_results_*_rank0.csv'.format(opts.output_dir))
top_files = Counter()
for f in prediction_files:
acc = judge(f, os.path.join(val_data_dir, 'answer.csv'))
top_files.update({f: acc})
print(top_files)
for f, acc in top_files.most_common(1):
m = pat.match(os.path.basename(f))
best_epoch = int(m.group('step'))
return best_epoch
def validate(opts, model, val_loader, split, global_step):
val_loss = 0
tot_score = 0
n_ex = 0
val_mrr = 0
st = time()
results = []
with tqdm(range(len(val_loader.dataset) // opts.size),
desc=f'{split}-{opts.rank}') as tq:
for i, batch in enumerate(val_loader):
qids = batch['qids']
targets = batch['targets']
del batch['targets']
del batch['qids']
del batch['gather_index']
scores, over_logits = model(**batch,
targets=None,
compute_loss=False)
loss = F.cross_entropy(scores, targets, reduction='sum')
val_loss += loss.item()
tot_score += (scores.max(
dim=-1, keepdim=False)[1] == targets).sum().item()
max_prob, max_idx = scores.max(dim=-1, keepdim=False)
answers = max_idx.cpu().tolist()
targets = torch.gather(batch['option_ids'],
dim=1,
index=targets.unsqueeze(1)).cpu().numpy()
for j, (qid, target) in enumerate(zip(qids, targets)):
g = over_logits[j].cpu().numpy()
top_k = np.argsort(-g)
val_mrr += 1 / (1 + np.argwhere(top_k == target).item())
results.extend(zip(qids, answers))
n_ex += len(qids)
tq.update(len(qids))
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}_rank{opts.rank}.csv'
with open(out_file, 'w') as f:
for id_, ans in results:
f.write(f'{id_},{ans}\n')
val_loss = sum(all_gather_list(val_loss))
val_mrr = sum(all_gather_list(val_mrr))
# tot_score = sum(all_gather_list(tot_score))
n_ex = sum(all_gather_list(n_ex))
tot_time = time() - st
val_loss /= n_ex
val_mrr = val_mrr / n_ex
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}.csv'
if not os.path.isfile(out_file):
with open(out_file, 'wb') as g:
for f in glob.glob(
f'{opts.output_dir}/results/{split}_results_{global_step}_rank*.csv'
):
shutil.copyfileobj(open(f, 'rb'), g)
sum(all_gather_list(opts.rank))
txt_db = os.path.join('/txt',
intermediate_dir(opts.pretrained_model_name_or_path),
getattr(opts, f'{split}_txt_db'))
val_acc = judge(out_file, f'{txt_db}/answer.csv')
val_log = {
f'{split}/loss': val_loss,
f'{split}/acc': val_acc,
f'{split}/mrr': val_mrr,
f'{split}/ex_per_s': n_ex / tot_time
}
LOGGER.info(f"validation finished in {int(tot_time)} seconds, "
f"score: {val_acc * 100:.2f}, "
f"mrr: {val_mrr:.3f}")
return val_log
def validate(opts, model, val_loader, split, global_step):
val_loss = 0
tot_score = 0
n_ex = 0
val_mrr = 0
st = time()
results = []
with tqdm(range(len(val_loader.dataset)),
desc=f'{split}-{opts.rank}') as tq:
for i, batch in enumerate(val_loader):
qids = batch['qids']
targets = batch['targets']
del batch['targets']
del batch['qids']
scores, over_logits, composition = model(**batch,
targets=None,
compute_loss=False)
loss = F.cross_entropy(scores, targets, reduction='sum')
val_loss += loss.item()
tot_score += (scores.max(
dim=-1, keepdim=False)[1] == targets).sum().item()
max_prob, max_idx = scores.max(dim=-1, keepdim=False)
select_masks, atts, composition_gates = composition
input_ids = torch.gather(batch['input_ids'],
dim=1,
index=batch['gather_index'])
targets = torch.gather(batch['option_ids'],
dim=1,
index=targets.unsqueeze(1)).cpu().numpy()
for j, (qid, target, inp, option_ids, position,
answer) in enumerate(
zip(qids, targets, input_ids, batch['option_ids'],
batch['positions'], max_idx)):
g = over_logits[j].cpu().numpy()
top_k = np.argsort(-g)
val_mrr += 1 / (1 + np.argwhere(top_k == target).item())
if i % 1000 == 0:
options = [
val_loader.dataset.id2idiom[o.item()]
for o in option_ids
]
idiom = options[answer.item()]
print(qid, val_loader.dataset.id2idiom[target.item()],
idiom, options)
print(len(select_masks), atts.size())
s_masks = [
select_mask[j].long().cpu().numpy().tolist()
for select_mask in select_masks
]
s_att = atts[j].cpu().numpy().tolist()
# tokens = val_loader.dataset.tokenizer.convert_ids_to_tokens(inp)
# start = tokens.index(val_loader.dataset.tokenizer.mask_token)
# tokens[position:position + len(idiom)] = list(idiom)
tokens = list(idiom)
print(tokens, s_masks, s_att, composition_gates[j].sum())
try:
tree = Tree(' '.join(tokens),
idiom2tree(tokens, s_masks))
print(TreePrettyPrinter(tree).text(unicodelines=True))
except:
pass
answers = max_idx.cpu().tolist()
results.extend(zip(qids, answers))
n_ex += len(qids)
tq.update(len(qids))
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}_rank{opts.rank}.csv'
with open(out_file, 'w') as f:
for id_, ans in results:
f.write(f'{id_},{ans}\n')
val_loss = sum(all_gather_list(val_loss))
val_mrr = sum(all_gather_list(val_mrr))
# tot_score = sum(all_gather_list(tot_score))
n_ex = sum(all_gather_list(n_ex))
tot_time = time() - st
val_loss /= n_ex
val_mrr = val_mrr / n_ex
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}.csv'
if not os.path.isfile(out_file):
with open(out_file, 'wb') as g:
for f in glob.glob(
f'{opts.output_dir}/results/{split}_results_{global_step}_rank*.csv'
):
shutil.copyfileobj(open(f, 'rb'), g)
sum(all_gather_list(opts.rank))
txt_db = os.path.join('/txt',
intermediate_dir(opts.pretrained_model_name_or_path),
getattr(opts, f'{split}_txt_db'))
val_acc = judge(out_file, f'{txt_db}/answer.csv')
val_log = {
f'{split}/loss': val_loss,
f'{split}/acc': val_acc,
f'{split}/mrr': val_mrr,
f'{split}/ex_per_s': n_ex / tot_time
}
LOGGER.info(f"validation finished in {int(tot_time)} seconds, "
f"score: {val_acc * 100:.2f}, "
f"mrr: {val_mrr:.3f}")
return val_log
def validate(opts, model, val_loader, split, global_step):
val_loss = 0
tot_score = 0
n_ex = 0
val_mrr = 0
st = time()
example_logits = {}
with open(f'{val_loader.dataset.db_dir}/id2eid.json', 'r') as f:
id2eid = json.load(f)
with tqdm(range(len(val_loader.dataset)), desc=split) as tq:
for i, batch in enumerate(val_loader):
qids = batch['qids']
targets = batch['targets']
del batch['targets']
del batch['gather_index']
del batch['qids']
logits, over_logits, cond_logits = model(**batch,
targets=None,
compute_loss=False)
loss = F.cross_entropy(logits, targets, reduction='sum')
val_loss += loss.item()
if opts.candidates == 'original':
logits = logits
elif opts.candidates == 'enlarged':
logits = cond_logits
elif opts.candidates == 'combined':
logits = logits + cond_logits
else:
raise AssertionError("No such loss!")
# scores, over_logits = model(**batch, targets=None, compute_loss=False)
# loss = F.cross_entropy(scores, targets, reduction='sum')
# val_loss += loss.item()
max_prob, max_idx = logits.max(dim=-1, keepdim=False)
tot_score += torch.eq(max_idx, targets).sum().item()
# tot_score += (scores.max(dim=-1, keepdim=False)[1] == targets).sum().item()
targets = torch.gather(batch['option_ids'],
dim=1,
index=targets.unsqueeze(1)).cpu().numpy()
for j, (qid, target, score, over_logit) in enumerate(
zip(qids, targets, logits, over_logits)):
g = over_logit.cpu().numpy()
top_k = np.argsort(-g)
val_mrr += 1 / (1 + np.argwhere(top_k == target).item())
eid = id2eid[qid]
example_logits.setdefault(eid, {})
example_logits[eid][qid] = score.cpu().numpy()
n_ex += len(qids)
tq.update(len(qids))
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}_rank{opts.rank}.csv'
with open(out_file, 'w') as f:
for id_, ans in optimize_answer(example_logits):
f.write(f'{id_},{ans}\n')
val_loss = sum(all_gather_list(val_loss))
n_ex = sum(all_gather_list(n_ex))
tot_time = time() - st
val_loss /= n_ex
val_mrr = val_mrr / n_ex
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}.csv'
if not os.path.isfile(out_file):
with open(out_file, 'wb') as g:
for f in glob.glob(
f'{opts.output_dir}/results/{split}_results_{global_step}_rank*.csv'
):
shutil.copyfileobj(open(f, 'rb'), g)
sum(all_gather_list(opts.rank))
txt_db = os.path.join('/txt',
intermediate_dir(opts.pretrained_model_name_or_path),
getattr(opts, f'{split}_txt_db'))
val_acc = judge(out_file, f'{txt_db}/answer.csv')
val_log = {
f'{split}/loss': val_loss,
f'{split}/acc': val_acc,
f'{split}/mrr': val_mrr,
f'{split}/ex_per_s': n_ex / tot_time
}
LOGGER.info(f"validation finished in {int(tot_time)} seconds, "
f"score: {val_acc * 100:.2f}, "
f"mrr: {val_mrr:.3f}")
return val_log
def validate_slide(opts, model, val_loader, split, global_step):
val_loss = 0
sentiment_score = 0
n_ex = 0
val_mrr = 0
st = time()
results = []
def get_header(key):
d = idioms_inverse_mapping[key]
return [f'{key}_{d[v]}_{v}' if isinstance(d[v], str) else f'{key}_{d[v][-1]}_{v}' for v in range(len(d))]
affection_results = []
with tqdm(range(len(val_loader.dataset) // opts.size), desc=f'{split}-{opts.rank}') as tq:
for i, batch in enumerate(val_loader):
qids = batch['qids']
targets = batch['targets']
del batch['targets']
del batch['qids']
# select_masks, atts, composition_gates = composition
if batch['input_ids'].dim() == 3:
input_ids = torch.gather(batch['input_ids'][1], dim=1, index=batch['gather_index'][0])
else:
input_ids = torch.gather(batch['input_ids'], dim=1, index=batch['gather_index'])
_, over_logits, select_masks, sentiment_logits = model(
**batch, targets=None, compute_loss=False)
idiom_targets = targets[:, 0]
sentiment_targets = targets[:, 1]
sentiment_score += (
sentiment_logits.max(dim=-1, keepdim=False)[1] == sentiment_targets).sum().item()
if over_logits is not None:
loss = F.cross_entropy(over_logits, idiom_targets, reduction='sum')
val_loss += loss.item()
# tot_score += (scores.max(dim=-1, keepdim=False)[1] == idiom_targets).sum().item()
max_prob, max_idx = over_logits.max(dim=-1, keepdim=False)
options = [val_loader.dataset.id2idiom[o] for o in val_loader.dataset.enlarged_candidates]
for j, (qid, inp, position, answer) in enumerate(zip(qids,
# idiom_targets,
input_ids,
# batch['option_ids'],
batch['positions'],
max_idx)):
# g = over_logits[j].cpu().numpy()
# top_k = np.argsort(-g)
# val_mrr += 1 / (1 + np.argwhere(top_k == target.item()).item())
example = val_loader.dataset.db[qid]
idiom = val_loader.dataset.id2idiom[example['idiom']]
# idiom = options[target.item()]
affection_results.append(
[idiom] + sentiment_logits[j].cpu().numpy().tolist()
)
if i % 1000 == 0:
g = over_logits[j].cpu().numpy()
top_k = np.argsort(-g)[:5]
print(qid,
[options[k] for k in top_k],
idiom)
# print(len(select_masks), atts.size())
if select_masks is not None:
s_masks = [select_mask[j].long().cpu().numpy().tolist() for select_mask in select_masks]
# s_att = atts[j].cpu().numpy().tolist()
# tokens = val_loader.dataset.tokenizer.convert_ids_to_tokens(inp)
# start = tokens.index(val_loader.dataset.tokenizer.mask_token)
# tokens[position:position + len(idiom)] = list(idiom)
tokens = val_loader.dataset.tokenizer.convert_ids_to_tokens(
val_loader.dataset.idiom_input_ids[qid])
# print(tokens, s_masks, s_att, composition_gates[j].sum())
print(tokens, s_masks)
try:
tree = Tree(' '.join(tokens), idiom2tree(tokens, s_masks))
print(TreePrettyPrinter(tree).text(unicodelines=True))
except:
pass
predictions = {
# "coarse emotion": {
# "target": calo_inverse_mapping['coarse_emotion'].get(coarse_emotion_targets[j].item(),
# '无'),
# "predictions": {calo_inverse_mapping['coarse_emotion'][k]: v for k, v in
# enumerate(coarse_emotion_logits[j].cpu().numpy().tolist())}
# },
"sentiment": {
"target": idioms_inverse_mapping['sentiment'].get(sentiment_targets[j].item(), '无'),
"predictions": {idioms_inverse_mapping['sentiment'][k]: v for k, v in
enumerate(sentiment_logits[j].cpu().numpy().tolist())}
}
}
pprint(predictions)
answers = max_idx.cpu().tolist()
results.extend(zip(qids, answers))
else:
for j, (qid, inp, position) in enumerate(zip(qids, input_ids,
# batch['option_ids'],
batch['positions'],
)):
# options = [val_loader.dataset.id2idiom[o.item()] for o in option_ids]
example = val_loader.dataset.db[qid]
idiom = val_loader.dataset.id2idiom[example['idiom']]
affection_results.append(
[idiom] + sentiment_logits[j].cpu().numpy().tolist()
)
if i % 1000 == 0:
print(qid,
idiom)
if select_masks is not None:
s_masks = [select_mask[j].long().cpu().numpy().tolist() for select_mask in select_masks]
tokens = val_loader.dataset.tokenizer.convert_ids_to_tokens(
val_loader.dataset.idiom_input_ids[qid])
# print(tokens, s_masks, s_att, composition_gates[j].sum())
print(tokens, s_masks)
try:
tree = Tree(' '.join(tokens), idiom2tree(tokens, s_masks))
print(TreePrettyPrinter(tree).text(unicodelines=True))
except:
pass
predictions = {
"sentiment": {
"target": idioms_inverse_mapping['sentiment'].get(sentiment_targets[j].item(), '无'),
"predictions": {idioms_inverse_mapping['sentiment'][k]: v for k, v in
enumerate(sentiment_logits[j].cpu().numpy().tolist())}
}
}
pprint(predictions)
n_ex += len(qids)
tq.update(len(qids))
if results:
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}_rank{opts.rank}.csv'
with open(out_file, 'w') as f:
for id_, ans in results:
f.write(f'{id_},{ans}\n')
header = ['idiom'] + get_header('sentiment')
if affection_results:
out_file = f'{opts.output_dir}/results/{split}_affection_results_{global_step}_rank{opts.rank}.csv'
pd.DataFrame(affection_results, columns=header).to_csv(out_file)
val_loss = sum(all_gather_list(val_loss))
val_mrr = sum(all_gather_list(val_mrr))
val_sentiment_score = sum(all_gather_list(sentiment_score))
n_ex = sum(all_gather_list(n_ex))
tot_time = time() - st
val_loss /= n_ex
val_mrr = val_mrr / n_ex
val_sentiment_score = val_sentiment_score / n_ex
if results:
out_file = f'{opts.output_dir}/results/{split}_results_{global_step}.csv'
if not os.path.isfile(out_file):
with open(out_file, 'wb') as g:
for f in glob.glob(f'{opts.output_dir}/results/{split}_results_{global_step}_rank*.csv'):
shutil.copyfileobj(open(f, 'rb'), g)
sum(all_gather_list(opts.rank))
txt_db = os.path.join('/txt',
intermediate_dir(opts.pretrained_model_name_or_path),
getattr(opts, f'{split}_txt_db'))
val_acc = judge(out_file, f'{txt_db}/answer.csv')
if opts.rank == 0:
results_files = glob.glob(f'{opts.output_dir}/results/{split}_affection_results_{global_step}_rank*.csv')
new_affection_results_df = pd.concat(map(pd.read_csv, results_files))
idiom_num = new_affection_results_df['idiom'].unique().size
idiom_wise_accs = {}
for item in new_affection_results_df.groupby('idiom').mean().reset_index().to_dict(orient='records'):
idiom = item['idiom']
idiom_id = val_loader.dataset.vocab[idiom]
for sub_type in ['sentiment']:
d = {k: v for k, v in item.items() if k.startswith(sub_type)}
key = max(d, key=d.get)
_, pred = key.rsplit('_', 1)
target = val_loader.dataset.sentiments[idiom_id]
idiom_wise_accs.setdefault(sub_type, 0)
idiom_wise_accs[sub_type] += (int(pred) == target) / idiom_num * 100
val_acc = val_sentiment_score
val_log = {f'{split}/loss': val_loss,
f'{split}/acc': val_acc,
f'{split}/sentiment': val_sentiment_score * 100,
f'{split}/mrr': val_mrr,
f'{split}/ex_per_s': n_ex / tot_time}
for k, v in idiom_wise_accs.items():
val_log[f'{split}/{k}'] = v
LOGGER.info(f"validation finished in {int(tot_time)} seconds, \n"
# f"coarse emotion score: {val_coarse_emotion_score * 100:.2f}, \n"
f"sentiment score: {val_sentiment_score * 100:.2f}, \n"
f"score: {val_acc * 100:.2f}, \n"
f"idiom-wise score: {idiom_wise_accs}, "
f"mrr: {val_mrr:.3f}")
return val_log