def error_analysis(sp): dataset = data_loader.load_processed_data(args) dev_examples = dataset['dev'] sp.schema_graphs = dataset['schema'] print('{} dev examples loaded'.format(len(dev_examples))) if len(ensemble_model_dirs) <= 2: print('Needs at least 3 models to perform majority vote') sys.exit() predictions = [] for model_dir in ensemble_model_dirs: pred_file = os.path.join(model_dir, 'predictions.16.txt') with open(pred_file) as f: predictions.append([x.strip() for x in f.readlines()]) for i in range(len(predictions)): assert(len(dev_examples) == len(predictions[i])) import collections disagree = collections.defaultdict(lambda: collections.defaultdict(list)) out_txt = 'majority_vote.txt' o_f = open(out_txt, 'w') for e_id in range(len(dev_examples)): example = dev_examples[e_id] gt_program_list = example.program_list votes = collections.defaultdict(list) for i in range(len(predictions)): pred_sql = predictions[i][e_id] votes[pred_sql].append(i) # break ties voting_results = sorted( votes.items(), key=lambda x: len(x[1]), reverse=True) voted_sql = voting_results[0][0] # TODO: the implementation below cheated # if len(voting_results) == 1: # voted_sql = voting_results[0][0] # else: # if len(voting_results[0][1]) > len(voting_results[1][1]): # voted_sql = voting_results[0][0] # else: # j = 1 # while(j < len(voting_results) and len(voting_results[j][1]) == len(voting_results[0][1])): # j += 1 # voting_results = sorted(voting_results[:j], key=lambda x:sum(x[1])) # voted_sql = voting_results[0][0] o_f.write(voted_sql + '\n') evals = [] for i in range(len(predictions)): eval_results, _, _ = eval_tools.eval_prediction( pred=predictions[i][e_id], gt_list=gt_program_list, dataset_id=example.dataset_id, db_name=example.db_name, in_execution_order=False ) evals.append(eval_results) models_agree = (len(set(evals)) == 1) if not models_agree: for i in range(len(evals)-1): for j in range(1, len(evals)): if evals[i] != evals[j]: disagree[i][j].append(e_id) schema = sp.schema_graphs[example.db_name] print('Example {}'.format(e_id+1)) example.pretty_print(schema) for i in range(len(predictions)): print('Prediction {} [{}]: {}'.format( i+1, evals[i], predictions[i][e_id])) print() o_f.close() for i in range(len(predictions)-1): for j in range(i+1, len(predictions)): print('Disagree {}, {}: {}'.format(i+1, j+1, len(disagree[i][j]))) import functools disagree_all = functools.reduce(lambda x, y: x & y, [set(l) for l in [ disagree[i][j] for i in range(len(disagree)) for j in disagree[i]]]) print('Disagree all: {}'.format(len(disagree_all))) print('Majority voting results saved to {}'.format(out_txt))
def inference(self, examples, decode_str_output=True, restore_clause_order=False, pred_restored_cache=None, check_schema_consistency_=True, engine=None, inline_eval=False, model_ensemble=None, verbose=False): # sanity check if self.args.leaderboard_submission or self.args.demo: assert (not verbose and not inline_eval and not self.args.use_oracle_tables) pred_list, pred_score_list, pred_decoded_list, pred_decoded_score_list = [], [], [], [] if restore_clause_order: if pred_restored_cache is None: pred_restored_cache = dict() if self.save_vis: text_ptr_weights_vis, pointer_vis = [], [] num_error_cases = 0 for batch_start_id in tqdm(range(0, len(examples), self.dev_batch_size)): mini_batch = examples[batch_start_id:batch_start_id + self.dev_batch_size] formatted_batch = self.format_batch(mini_batch) outputs = self.forward(formatted_batch, model_ensemble) if self.model_id in [SEQ2SEQ_PG, BRIDGE]: preds, pred_scores, text_p_pointers, text_ptr_weights, seq_len = outputs text_p_pointers.unsqueeze_(2) p_pointers = torch.cat([1 - text_p_pointers, text_p_pointers], dim=2) elif self.model_id == SEQ2SEQ: preds, pred_scores, text_ptr_weights, seq_len = outputs p_pointers = None else: raise NotImplementedError pred_list.append(preds) pred_score_list.append(pred_scores) if decode_str_output or verbose: for i in range(len(mini_batch)): example = mini_batch[i] db_name = example.db_name schema = self.schema_graphs[db_name] table_po, field_po = None, None if self.args.use_oracle_tables: # TODO: The implementation below is incorrect. if self.args.num_random_tables_added > 0: table_po, field_po = formatted_batch[-1][i] exp_output_strs, exp_output_scores, exp_seq_lens, exp_correct = [], [], [], [] if inline_eval: if example.dataset_id == SPIDER: gt_program_list = example.program_list gt_program_ast = example.program_ast_list_[0] \ if example.program_ast_list_ else example.program hardness = spider_eval_tools.Evaluator( ).eval_hardness(gt_program_ast, db_dir=self.args.db_dir, db_name=example.db_name) elif example.dataset_id == WIKISQL: gt_program_list = example.program_ast_list_ else: raise NotImplementedError if example.dataset_id == WIKISQL: hardness = 'easy' if self.decoding_algorithm == 'beam-search': for j in range(self.beam_size): beam_id = i * self.beam_size + j post_processed_output = self.post_process_nn_output( beam_id, example.dataset_id, example, preds, schema, text_ptr_weights, p_pointers, table_po=table_po, field_po=field_po, verbose=verbose) if post_processed_output: pred_sql = post_processed_output[0] # print('{}\t{}'.format(pred_sql, float(pred_scores[beam_id]))) if restore_clause_order: if pred_restored_cache and db_name in pred_restored_cache and \ pred_sql in pred_restored_cache[db_name]: restored_pred, grammatical, schema_consistent = pred_restored_cache[ db_name][pred_sql] else: restored_pred, grammatical, schema_consistent = moz_sp.restore_clause_order( pred_sql, schema, check_schema_consistency_= check_schema_consistency_, verbose=verbose) if pred_restored_cache and check_schema_consistency_: # TODO: we don't cache the results when check_schema_consistency_ is off to # avoid logging false negatives if db_name not in pred_restored_cache: pred_restored_cache[ db_name] = dict() pred_restored_cache[db_name][pred_sql] = restored_pred, grammatical, \ schema_consistent if check_schema_consistency_ and not schema_consistent: restored_pred = None pred_sql = restored_pred else: if check_schema_consistency_: if not moz_sp.check_schema_consistency( pred_sql, schema, in_execution_order=self.args. process_sql_in_execution_order ): pred_sql = None if pred_sql and self.args.execution_guided_decoding: assert (engine is not None) try: pred_query = Query.from_dict( pred_sql, ordered=False) pred_ex = engine.execute_query( example.db_name, pred_query, lower=True) if not pred_ex: pred_sql = None except Exception: pred_sql = None else: pred_sql = None # if not pred_sql: # pred_sql = self.get_dummy_prediction(schema) if pred_sql: exp_output_strs.append(pred_sql) exp_output_scores.append( float(pred_scores[beam_id])) exp_seq_lens.append(int(seq_len[beam_id])) if self.save_vis: self.save_vis_parameters( post_processed_output, text_ptr_weights_vis, pointer_vis) if inline_eval: results = eval_tools.eval_prediction( pred=pred_sql, gt_list=gt_program_list, dataset_id=example.dataset_id, db_name=example.db_name, in_execution_order=( self.args. process_sql_in_execution_order and not restore_clause_order)) correct, _, _ = results exp_correct.append(correct) correct_ = correct[1] if isinstance( correct, tuple) else correct if correct_: break else: raise NotImplementedError num_preds = len(exp_output_strs) pred_decoded_list.append(exp_output_strs) pred_decoded_score_list.append( exp_output_scores[:num_preds]) if verbose: predictions = zip(exp_output_strs, exp_output_scores, exp_seq_lens, exp_correct) is_error_case = self.print_predictions( batch_start_id + i, example, hardness, predictions, schema) if is_error_case: num_error_cases += 1 print('Error Case {}'.format(num_error_cases)) print() # if num_error_cases == 50: # import sys # sys.exit() if not pred_decoded_list[-1] and not self.args.demo: pred_decoded_list[-1].append( self.get_dummy_prediction(schema)) pred_decoded_score_list[-1].append(-ops.HUGE_INT) out_dict = dict() out_dict['preds'] = ops.pad_and_cat(pred_list, self.out_vocab.pad_id) out_dict['pred_scores'] = torch.cat(pred_score_list) if decode_str_output: out_dict['pred_decoded'] = pred_decoded_list out_dict['pred_decoded_scores'] = pred_decoded_score_list if restore_clause_order: out_dict['pred_restored_cache'] = pred_restored_cache if self.save_vis: vis_dict = dict() vis_dict['text_attention_vis'] = text_ptr_weights_vis vis_dict['text_pointer_vis'] = pointer_vis for key in vis_dict: if key.endswith('_vis'): if key.endswith('_attention_vis'): attn_target_label = key.split('_')[0] self.vis_writer.save_cross_attention( vis_dict[key], attn_target_label) if key.endswith('_pointer_vis'): self.vis_writer.save_pointer(vis_dict[key], 'all') return out_dict