def combine_annotations(): """ Combine the annotations input by three annotators. :param input_file1: main annotation file 1. :param input_file2: main annotation file 2 (should contain the same number of lines as input_file1). :param input_file3: supplementary annotation file which contains annotations of lines in input_file1 and input_file2 that contain a disagreement. :param output_file: file that contains the combined annotations. """ input_file1 = sys.argv[1] input_file2 = sys.argv[2] input_file3 = sys.argv[3] output_file = sys.argv[4] o_f = open(output_file, 'w') o_f.write('description,prediction,template,correct template,correct command,' 'correct template A,correct command A,' 'correct template B,correct command B,' 'correct template C,correct command C\n') sup_structure_eval, sup_command_eval = load_cached_evaluations_from_file( input_file3, treat_empty_as_correct=True) # for key in sup_structure_eval: # print(key) # print('------------------') with open(input_file1) as f1: with open(input_file2) as f2: reader1 = csv.DictReader(f1) reader2 = csv.DictReader(f2) current_desp = '' for row1, row2 in zip(reader1, reader2): row1_template_eval = normalize_judgement(row1['correct template'].strip()) row1_command_eval = normalize_judgement(row1['correct command'].strip()) row2_template_eval = normalize_judgement(row2['correct template'].strip()) row2_command_eval = normalize_judgement(row2['correct command'].strip()) if row1['description']: current_desp = row1['description'].strip() sc_key = get_example_nl_key(current_desp) pred_cmd = row1['prediction'].strip() if not pred_cmd: row1_template_eval, row1_command_eval = 'n', 'n' row2_template_eval, row2_command_eval = 'n', 'n' pred_temp = data_tools.cmd2template(pred_cmd, loose_constraints=True) structure_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_temp) command_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_cmd) row3_template_eval, row3_command_eval = None, None if structure_example_key in sup_structure_eval: row3_template_eval = sup_structure_eval[structure_example_key] if command_example_key in sup_command_eval: row3_command_eval = sup_command_eval[command_example_key] if row1_template_eval != row2_template_eval or row1_command_eval != row2_command_eval: if row1_template_eval != row2_template_eval: if row3_template_eval is None: print(structure_example_key) assert(row3_template_eval is not None) template_eval = row3_template_eval else: template_eval = row1_template_eval if row1_command_eval != row2_command_eval: # if row3_command_eval is None: # print(command_example_key) assert(row3_command_eval is not None) command_eval = row3_command_eval else: command_eval = row1_command_eval else: template_eval = row1_template_eval command_eval = row1_command_eval if row3_template_eval is None: row3_template_eval = '' if row3_command_eval is None: row3_command_eval = '' o_f.write('"{}","{}","{}",{},{},{},{},{},{},{},{}\n'.format( current_desp.replace('"', '""'), pred_cmd.replace('"', '""'), pred_temp.replace('"', '""'), template_eval, command_eval, row1_template_eval, row1_command_eval, row2_template_eval, row2_command_eval, row3_template_eval, row3_command_eval)) o_f.close()
def print_error_analysis_sheet(): input_file1 = sys.argv[1] input_file2 = sys.argv[2] input_file3 = sys.argv[3] output_file = sys.argv[4] o_f = open(output_file, 'w') o_f.write('description,model,prediction,correct template,correct command,' 'correct template A,correct command A,' 'correct template B,correct command B,' 'correct template C,correct command C\n') sup_structure_eval, sup_command_eval = load_cached_evaluations_from_file( input_file3, treat_empty_as_correct=True) # for key in sup_structure_eval: # print(key) # print('------------------') with open(input_file1) as f1: with open(input_file2) as f2: reader1 = csv.DictReader(f1) reader2 = csv.DictReader(f2) current_desp = '' for row_id, (row1, row2) in enumerate(zip(reader1, reader2)): if row1['description']: current_desp = row1['description'].strip() model_name = row2['model'] if not model_name in ['partial.token-copynet', 'tellina']: continue if row_id % 3 != 0: continue row1_template_eval = normalize_judgement(row1['correct template'].strip()) row1_command_eval = normalize_judgement(row1['correct command'].strip()) row2_template_eval = normalize_judgement(row2['correct template'].strip()) row2_command_eval = normalize_judgement(row2['correct command'].strip()) sc_key = get_example_nl_key(current_desp) pred_cmd = row1['prediction'].strip() if not pred_cmd: row1_template_eval, row1_command_eval = 'n', 'n' row2_template_eval, row2_command_eval = 'n', 'n' pred_temp = data_tools.cmd2template(pred_cmd, loose_constraints=True) structure_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_temp) command_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_cmd) row3_template_eval, row3_command_eval = None, None if structure_example_key in sup_structure_eval: row3_template_eval = sup_structure_eval[structure_example_key] if command_example_key in sup_command_eval: row3_command_eval = sup_command_eval[command_example_key] if row1_template_eval != row2_template_eval or row1_command_eval != row2_command_eval: if row1_template_eval != row2_template_eval: if row3_template_eval is None: print(pred_cmd_key, structure_example_key) assert (row3_template_eval is not None) template_eval = row3_template_eval else: template_eval = row1_template_eval if row1_command_eval != row2_command_eval: # if row3_command_eval is None: # print(command_example_key) assert (row3_command_eval is not None) command_eval = row3_command_eval else: command_eval = row1_command_eval else: template_eval = row1_template_eval command_eval = row1_command_eval if row3_template_eval is None: row3_template_eval = '' if row3_command_eval is None: row3_command_eval = '' o_f.write('"{}","{}","{}",{},{},{},{},{},{},{},{}\n'.format( current_desp.replace('"', '""'), model_name, pred_cmd.replace('"', '""'), template_eval, command_eval, row1_template_eval, row1_command_eval, row2_template_eval, row2_command_eval, row3_template_eval, row3_command_eval)) o_f.close()