def dump_example(subtask, split):
test_data = prepare_data(subtask, split)
pred = extract_gt(test_data)
subdir = get_subdir(subtask)
json.dump(pred,
open(os.path.join('example', subdir, 'submission1.json'), 'w'),
ensure_ascii=False,
indent=4)
import random
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot, value in domain.items():
if value:
if random.randint(0, 2) == 0:
domain[slot] = ""
else:
if random.randint(0, 4) == 0:
domain[slot] = "2333"
json.dump(pred,
open(os.path.join('example', subdir, 'submission2.json'), 'w'),
ensure_ascii=False,
indent=4)
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(pred,
open(os.path.join('example', subdir, 'submission3.json'), 'w'),
ensure_ascii=False,
indent=4)
def dump_example(subtask, split):
test_data = prepare_data(subtask, split)
gt = extract_gt(test_data)
json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission1.json'), 'w'), ensure_ascii=False, indent=4)
for dialog_id, states in gt.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission2.json'), 'w'), ensure_ascii=False, indent=4)
def eval_team(team, correct_name_label):
for subtask in ['multiwoz', 'crosswoz']:
test_data = prepare_data(subtask,
'dstc9',
correct_name_label=correct_name_label)
gt = extract_gt(test_data)
for i in range(1, 6):
model_dir = os.path.join(team, f'{subtask}-dst', f'submission{i}')
if not os.path.exists(model_dir):
continue
print(model_dir)
evaluate(model_dir, subtask, test_data, gt)
"""
evaluate output file
"""
from argparse import ArgumentParser
import os
from convlab2.dst.dstc9.utils import prepare_data, extract_gt
from convlab2.dst.dstc9.eval_file import evaluate
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument('--correct_name_label', action='store_true')
args = parser.parse_args()
for subtask in ['multiwoz', 'crosswoz']:
test_data = prepare_data(subtask,
'dstc9-250',
correct_name_label=args.correct_name_label)
gt = extract_gt(test_data)
for team in os.listdir('.'):
for i in range(1, 6):
model_dir = os.path.join(team, f'{subtask}-dst',
f'submission{i}')
if not os.path.exists(model_dir):
continue
print(model_dir)
evaluate(model_dir, subtask, gt)
pred = {}
for dialog_id, turns in test_data.items():
model.init_session()
pred[dialog_id] = [
model.update_turn(sys_utt, user_utt)
for sys_utt, user_utt, gt_turn in turns
]
result = eval_states(gt, pred)
print(result)
json.dump(result,
open(os.path.join(model_dir, subdir, 'model-result.json'), 'w'),
indent=4,
ensure_ascii=False)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('subtask', type=str, choices=['multiwoz', 'crosswoz'])
parser.add_argument('split',
type=str,
choices=['train', 'val', 'test', 'human_val'])
args = parser.parse_args()
subtask = args.subtask
test_data = prepare_data(subtask, args.split)
gt = {
dialog_id: [state for _, _, state in turns]
for dialog_id, turns in test_data.items()
}
evaluate('example', subtask, test_data, gt)
'submission2.json'), 'w'),
ensure_ascii=False,
indent=4)
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(pred,
open(
os.path.join('example', get_subdir(subtask),
'submission3.json'), 'w'),
ensure_ascii=False,
indent=4)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('subtask', type=str, choices=['multiwoz', 'crosswoz'])
parser.add_argument('split',
type=str,
choices=['train', 'val', 'test', 'human_val'])
args = parser.parse_args()
subtask = args.subtask
split = args.split
dump_example(subtask, split)
test_data = prepare_data(subtask, split)
gt = extract_gt(test_data)
evaluate('example', subtask, gt)
for state in states:
for domain in state.values():
for slot, value in domain.items():
if value:
if random.randint(0, 2) == 0:
domain[slot] = ""
else:
if random.randint(0, 4) == 0:
domain[slot] = "2333"
json.dump(pred, open(os.path.join('example', subdir, 'submission2.json'), 'w'), ensure_ascii=False, indent=4)
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(pred, open(os.path.join('example', subdir, 'submission3.json'), 'w'), ensure_ascii=False, indent=4)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('subtask', type=str, choices=['multiwoz', 'crosswoz'])
parser.add_argument('split', type=str, choices=['train', 'val', 'test', 'human_val', 'dstc9-250'])
parser.add_argument('correct_name_label', action='store_true')
args = parser.parse_args()
subtask = args.subtask
split = args.split
dump_example(subtask, split)
test_data = prepare_data(subtask, split, correct_name_label=args.correct_name_label)
gt = extract_gt(test_data)