def dump_example(subtask, split): test_data = prepare_data(subtask, split) pred = extract_gt(test_data) subdir = get_subdir(subtask) json.dump(pred, open(os.path.join('example', subdir, 'submission1.json'), 'w'), ensure_ascii=False, indent=4) import random for dialog_id, states in pred.items(): for state in states: for domain in state.values(): for slot, value in domain.items(): if value: if random.randint(0, 2) == 0: domain[slot] = "" else: if random.randint(0, 4) == 0: domain[slot] = "2333" json.dump(pred, open(os.path.join('example', subdir, 'submission2.json'), 'w'), ensure_ascii=False, indent=4) for dialog_id, states in pred.items(): for state in states: for domain in state.values(): for slot in domain: domain[slot] = "" json.dump(pred, open(os.path.join('example', subdir, 'submission3.json'), 'w'), ensure_ascii=False, indent=4)
def dump_example(subtask, split): test_data = prepare_data(subtask, split) gt = extract_gt(test_data) json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission1.json'), 'w'), ensure_ascii=False, indent=4) for dialog_id, states in gt.items(): for state in states: for domain in state.values(): for slot in domain: domain[slot] = "" json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission2.json'), 'w'), ensure_ascii=False, indent=4)
def eval_team(team, correct_name_label): for subtask in ['multiwoz', 'crosswoz']: test_data = prepare_data(subtask, 'dstc9', correct_name_label=correct_name_label) gt = extract_gt(test_data) for i in range(1, 6): model_dir = os.path.join(team, f'{subtask}-dst', f'submission{i}') if not os.path.exists(model_dir): continue print(model_dir) evaluate(model_dir, subtask, test_data, gt)
""" evaluate output file """ from argparse import ArgumentParser import os from convlab2.dst.dstc9.utils import prepare_data, extract_gt from convlab2.dst.dstc9.eval_file import evaluate if __name__ == '__main__': parser = ArgumentParser() parser.add_argument('--correct_name_label', action='store_true') args = parser.parse_args() for subtask in ['multiwoz', 'crosswoz']: test_data = prepare_data(subtask, 'dstc9-250', correct_name_label=args.correct_name_label) gt = extract_gt(test_data) for team in os.listdir('.'): for i in range(1, 6): model_dir = os.path.join(team, f'{subtask}-dst', f'submission{i}') if not os.path.exists(model_dir): continue print(model_dir) evaluate(model_dir, subtask, gt)