def combine_annotations():
"""
Combine the annotations input by three annotators.
:param input_file1: main annotation file 1.
:param input_file2: main annotation file 2 (should contain the same number of
lines as input_file1).
:param input_file3: supplementary annotation file which contains annotations
of lines in input_file1 and input_file2 that contain a disagreement.
:param output_file: file that contains the combined annotations.
"""
input_file1 = sys.argv[1]
input_file2 = sys.argv[2]
input_file3 = sys.argv[3]
output_file = sys.argv[4]
o_f = open(output_file, 'w')
o_f.write('description,prediction,template,correct template,correct command,'
'correct template A,correct command A,'
'correct template B,correct command B,'
'correct template C,correct command C\n')
sup_structure_eval, sup_command_eval = load_cached_evaluations_from_file(
input_file3, treat_empty_as_correct=True)
# for key in sup_structure_eval:
# print(key)
# print('------------------')
with open(input_file1) as f1:
with open(input_file2) as f2:
reader1 = csv.DictReader(f1)
reader2 = csv.DictReader(f2)
current_desp = ''
for row1, row2 in zip(reader1, reader2):
row1_template_eval = normalize_judgement(row1['correct template'].strip())
row1_command_eval = normalize_judgement(row1['correct command'].strip())
row2_template_eval = normalize_judgement(row2['correct template'].strip())
row2_command_eval = normalize_judgement(row2['correct command'].strip())
if row1['description']:
current_desp = row1['description'].strip()
sc_key = get_example_nl_key(current_desp)
pred_cmd = row1['prediction'].strip()
if not pred_cmd:
row1_template_eval, row1_command_eval = 'n', 'n'
row2_template_eval, row2_command_eval = 'n', 'n'
pred_temp = data_tools.cmd2template(pred_cmd, loose_constraints=True)
structure_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_temp)
command_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_cmd)
row3_template_eval, row3_command_eval = None, None
if structure_example_key in sup_structure_eval:
row3_template_eval = sup_structure_eval[structure_example_key]
if command_example_key in sup_command_eval:
row3_command_eval = sup_command_eval[command_example_key]
if row1_template_eval != row2_template_eval or row1_command_eval != row2_command_eval:
if row1_template_eval != row2_template_eval:
if row3_template_eval is None:
print(structure_example_key)
assert(row3_template_eval is not None)
template_eval = row3_template_eval
else:
template_eval = row1_template_eval
if row1_command_eval != row2_command_eval:
# if row3_command_eval is None:
# print(command_example_key)
assert(row3_command_eval is not None)
command_eval = row3_command_eval
else:
command_eval = row1_command_eval
else:
template_eval = row1_template_eval
command_eval = row1_command_eval
if row3_template_eval is None:
row3_template_eval = ''
if row3_command_eval is None:
row3_command_eval = ''
o_f.write('"{}","{}","{}",{},{},{},{},{},{},{},{}\n'.format(
current_desp.replace('"', '""'), pred_cmd.replace('"', '""'), pred_temp.replace('"', '""'),
template_eval, command_eval,
row1_template_eval, row1_command_eval,
row2_template_eval, row2_command_eval,
row3_template_eval, row3_command_eval))
o_f.close()
def print_error_analysis_sheet():
input_file1 = sys.argv[1]
input_file2 = sys.argv[2]
input_file3 = sys.argv[3]
output_file = sys.argv[4]
o_f = open(output_file, 'w')
o_f.write('description,model,prediction,correct template,correct command,'
'correct template A,correct command A,'
'correct template B,correct command B,'
'correct template C,correct command C\n')
sup_structure_eval, sup_command_eval = load_cached_evaluations_from_file(
input_file3, treat_empty_as_correct=True)
# for key in sup_structure_eval:
# print(key)
# print('------------------')
with open(input_file1) as f1:
with open(input_file2) as f2:
reader1 = csv.DictReader(f1)
reader2 = csv.DictReader(f2)
current_desp = ''
for row_id, (row1, row2) in enumerate(zip(reader1, reader2)):
if row1['description']:
current_desp = row1['description'].strip()
model_name = row2['model']
if not model_name in ['partial.token-copynet', 'tellina']:
continue
if row_id % 3 != 0:
continue
row1_template_eval = normalize_judgement(row1['correct template'].strip())
row1_command_eval = normalize_judgement(row1['correct command'].strip())
row2_template_eval = normalize_judgement(row2['correct template'].strip())
row2_command_eval = normalize_judgement(row2['correct command'].strip())
sc_key = get_example_nl_key(current_desp)
pred_cmd = row1['prediction'].strip()
if not pred_cmd:
row1_template_eval, row1_command_eval = 'n', 'n'
row2_template_eval, row2_command_eval = 'n', 'n'
pred_temp = data_tools.cmd2template(pred_cmd, loose_constraints=True)
structure_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_temp)
command_example_key = '{}<NL_PREDICTION>{}'.format(sc_key, pred_cmd)
row3_template_eval, row3_command_eval = None, None
if structure_example_key in sup_structure_eval:
row3_template_eval = sup_structure_eval[structure_example_key]
if command_example_key in sup_command_eval:
row3_command_eval = sup_command_eval[command_example_key]
if row1_template_eval != row2_template_eval or row1_command_eval != row2_command_eval:
if row1_template_eval != row2_template_eval:
if row3_template_eval is None:
print(pred_cmd_key, structure_example_key)
assert (row3_template_eval is not None)
template_eval = row3_template_eval
else:
template_eval = row1_template_eval
if row1_command_eval != row2_command_eval:
# if row3_command_eval is None:
# print(command_example_key)
assert (row3_command_eval is not None)
command_eval = row3_command_eval
else:
command_eval = row1_command_eval
else:
template_eval = row1_template_eval
command_eval = row1_command_eval
if row3_template_eval is None:
row3_template_eval = ''
if row3_command_eval is None:
row3_command_eval = ''
o_f.write('"{}","{}","{}",{},{},{},{},{},{},{},{}\n'.format(
current_desp.replace('"', '""'), model_name, pred_cmd.replace('"', '""'),
template_eval, command_eval,
row1_template_eval, row1_command_eval,
row2_template_eval, row2_command_eval,
row3_template_eval, row3_command_eval))
o_f.close()