def perturb_interp(args, model, tokenizer, prefix=""):
if not os.path.exists(args.interp_dir):
os.makedirs(args.interp_dir)
# fix the model
model.requires_grad_(False)
dataset, examples, features = load_and_cache_examples(args,
tokenizer,
evaluate=True,
output_examples=True)
# assume one on on mapping
assert len(examples) == len(features)
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
args.eval_batch_size = 1
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
all_predictions = []
start_time = timeit.default_timer()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
feature_indices = to_list(batch[3])
batch_features = [features[i] for i in feature_indices]
batch_examples = [examples[i] for i in feature_indices]
# batch prem, batch predictions
batch = remove_padding(batch, batch_features[0])
batch_predictions, batch_prelim_results, batch_attributions = predict_and_fit_locality(
args, batch, model, tokenizer, batch_features, batch_examples)
dump_token_interp_info(args, batch_examples, batch_features, tokenizer,
batch_predictions, batch_prelim_results,
batch_attributions)
# lots of info, dump to files immediately
all_predictions.append(batch_predictions)
evalTime = timeit.default_timer() - start_time
logger.info(" Evaluation done in total %f secs (%f sec per example)",
evalTime, evalTime / len(dataset))
all_predictions = merge_predictions(all_predictions)
results = hotpot_evaluate(examples[:len(all_predictions)], all_predictions)
return results
def attention_interp(args, model, tokenizer, prefix=""):
if not os.path.exists(args.interp_dir):
os.makedirs(args.interp_dir)
dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)
# fix the model
model.requires_grad_(False)
# assume one on on mapping
assert len(examples) == len(features)
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
# restrict evak batch size
assert args.per_gpu_eval_batch_size == 1 and args.n_gpu <= 1
args.eval_batch_size = 1
# Note that DistributedSampler samples randomly
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
all_predictions = []
start_time = timeit.default_timer()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
feature_indices = to_list(batch[3])
batch_features = [features[i] for i in feature_indices]
batch_examples = [examples[i] for i in feature_indices]
# batch prem, batch predictions
batch = remove_padding(batch, batch_features[0])
batch_predictions, batch_prelim_results, batch_attentions, batch_attributions = predict_and_layerwise_attribute(
args,
batch,
model,
tokenizer,
batch_features,
batch_examples
)
# lots of info, dump to files immediately
dump_attention_interp_info(args, batch_examples, batch_features, tokenizer, batch_predictions, batch_prelim_results, batch_attentions, batch_attributions)
all_predictions.append(batch_predictions)
evalTime = timeit.default_timer() - start_time
logger.info(" Evaluation done in total %f secs (%f sec per example)", evalTime, evalTime / len(dataset))
# Compute predictions
# output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix))
# output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix))
# Compute the F1 and exact scores.
all_predictions = merge_predictions(all_predictions)
results = hotpot_evaluate(examples[:len(all_predictions)], all_predictions)
return results