def dump_example(subtask, split):
test_data = prepare_data(subtask, split)
gt = extract_gt(test_data)
json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission1.json'), 'w'), ensure_ascii=False, indent=4)
for dialog_id, states in gt.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(gt, open(os.path.join('example', get_subdir(subtask), 'submission2.json'), 'w'), ensure_ascii=False, indent=4)
def dump_example(subtask, split):
test_data = prepare_data(subtask, split)
pred = extract_gt(test_data)
subdir = get_subdir(subtask)
json.dump(pred,
open(os.path.join('example', subdir, 'submission1.json'), 'w'),
ensure_ascii=False,
indent=4)
import random
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot, value in domain.items():
if value:
if random.randint(0, 2) == 0:
domain[slot] = ""
else:
if random.randint(0, 4) == 0:
domain[slot] = "2333"
json.dump(pred,
open(os.path.join('example', subdir, 'submission2.json'), 'w'),
ensure_ascii=False,
indent=4)
for dialog_id, states in pred.items():
for state in states:
for domain in state.values():
for slot in domain:
domain[slot] = ""
json.dump(pred,
open(os.path.join('example', subdir, 'submission3.json'), 'w'),
ensure_ascii=False,
indent=4)
def evaluate(model_dir, subtask, gt):
subdir = get_subdir(subtask)
results = {}
for i in range(1, 6):
filepath = os.path.join(model_dir, subdir, f'submission{i}.json')
if not os.path.exists(filepath):
continue
pred = json.load(open(filepath))
results[i] = eval_states(gt, pred)
json.dump(results, open(os.path.join(model_dir, subdir, 'file-results.json'), 'w'), indent=4, ensure_ascii=False)
def evaluate(model_dir, subtask, test_data, gt):
subdir = get_subdir(subtask)
module = importlib.import_module(f'{model_dir}.{subdir}')
assert 'Model' in dir(
module
), 'please import your model as name `Model` in your subtask module root'
model_cls = module.__getattribute__('Model')
assert issubclass(model_cls, DST), 'the model must implement DST interface'
# load weights, set eval() on default
model = model_cls()
pred = {}
for dialog_id, turns in test_data.items():
model.init_session()
pred[dialog_id] = [
model.update_turn(sys_utt, user_utt)
for sys_utt, user_utt, gt_turn in turns
]
result = eval_states(gt, pred)
print(result)
json.dump(result,
open(os.path.join(model_dir, subdir, 'model-result.json'), 'w'),
indent=4,
ensure_ascii=False)