def test_create_programs_with_vanilla_salience_model(data_dir: str):
"""Tests creating programs with a vanilla salience model.
"""
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
expected_num_refer_calls = {
"MUL1626.json": 1,
"PMUL3166.json": 0,
"MUL2258.json": 1,
"MUL2199.json": 1,
"MUL2096.json": 0,
"PMUL3470.json": 0,
"PMUL4478.json": 0,
}
for trade_dialogue in load_test_trade_dialogues(data_dir):
dataflow_dialogue, num_refer_calls, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
dialogue_id = dataflow_dialogue.dialogue_id
assert (num_refer_calls == expected_num_refer_calls[dialogue_id]
), "{} failed".format(dialogue_id)
def test_create_programs_with_revise_with_fill_none(
trade_dialogue_1: Dict[str, Any]):
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
expected_plans: List[str] = [
# turn 1
"""(find (Constraint[Hotel] :area (?= #(String "none")) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-stay (?= #(String "none")) :internet (?= #(String "none")) :name (?= #(String "none")) :parking (?= #(String "none")) :pricerange (?= #(String "none")) :stars (?= #(String "none")) :type (?= #(String "none"))))""",
# turn 2
"""(ReviseConstraint :new (Constraint[Hotel] :name (?= #(String "hilton")) :pricerange (?= #(String "cheap")) :type (?= #(String "guest house"))) :oldLocation (Constraint[Constraint[Hotel]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 3
"""(ReviseConstraint :new (Constraint[Hotel] :name (?= #(String "none"))) :oldLocation (Constraint[Constraint[Hotel]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 4
"""(abandon (Constraint[Hotel]))""",
# turn 5
"""(find (Constraint[Hotel] :area (?= #(String "west")) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-stay (?= #(String "none")) :internet (?= #(String "none")) :name (?= #(String "none")) :parking (?= #(String "none")) :pricerange (?= #(String "none")) :stars (?= #(String "none")) :type (?= #(String "none"))))""",
# turn 6
"""(find (Constraint[Restaurant] :area (refer (Constraint[Area])) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-time (?= #(String "none")) :food (?= #(String "none")) :name (?= #(String "none")) :pricerange (?= #(String "none"))))""",
# turn 7
"""(ReviseConstraint :new (Constraint[Restaurant] :pricerange (refer (Constraint[Pricerange]))) :oldLocation (Constraint[Constraint[Restaurant]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 8
"()",
# turn 9
"""(find (Constraint[Taxi] :arriveby (?= #(String "none")) :departure (?= #(String "none")) :destination (?= #(String "none")) :leaveat (?= #(String "none"))))""",
# turn 10
"()",
]
dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue_1,
keep_all_domains=True,
remove_none=False,
fill_none=True,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
for turn, expected_plan in zip(
dataflow_dialogue.turns,
expected_plans # pylint: disable=no-member
):
lispress = turn.lispress
assert lispress == expected_plan
def test_create_programs_without_revise(data_dir: str):
"""Tests creating programs without revise calls.
It should not use refer calls even with a valid salience model.
"""
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
for trade_dialogue in load_test_trade_dialogues(data_dir):
for avoid_empty_plan in [True, False]:
_, num_refer_calls, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=True,
avoid_empty_plan=avoid_empty_plan,
utterance_tokenizer=utterance_tokenizer,
)
assert num_refer_calls == 0
def main(
dialogues_file: str,
no_refer: bool,
no_revise: bool,
cheating_mode: str,
cheating_execution_results_file: Optional[str],
outbase: str,
) -> Tuple[str, str, str]:
salience_model: SalienceModelBase
if no_refer:
salience_model = DummySalienceModel()
else:
salience_model = VanillaSalienceModel()
cheating_execution_results_lookup = None
if cheating_execution_results_file is not None:
cheating_execution_results_lookup = load_jsonl_file_and_build_lookup(
data_jsonl=cheating_execution_results_file,
cls=CompleteExecutionResult,
primary_key_getter=lambda x: x.dialogue_id,
secondary_key_getter=lambda x: x.turn_index,
)
complete_execution_results_file = outbase + ".execution_results.jsonl"
cheating_report_file = outbase + ".cheating_report.jsonl"
complete_execution_results_fp = open(complete_execution_results_file, "w")
cheating_report_fp = open(cheating_report_file, "w")
for dialogue in load_jsonl_file(
data_jsonl=dialogues_file, cls=Dialogue, unit=" dialogues"
):
if cheating_execution_results_lookup is None:
cheating_execution_results = None
else:
cheating_execution_results = cheating_execution_results_lookup.get(
dialogue.dialogue_id
)
assert cheating_execution_results is not None
(
complete_execution_results,
cheating_turn_indices,
) = execute_programs_for_dialogue(
dialogue=dialogue,
salience_model=salience_model,
no_revise=no_revise,
cheating_mode=cheating_mode,
cheating_execution_results=cheating_execution_results,
)
for complete_execution_result in complete_execution_results:
complete_execution_results_fp.write(jsons.dumps(complete_execution_result))
complete_execution_results_fp.write("\n")
num_total_turns = len(dialogue.turns)
assert (
dialogue.turns[-1].turn_index - dialogue.turns[0].turn_index + 1
== num_total_turns
)
num_cheating_turns = len(cheating_turn_indices)
cheating_report_fp.write(
json.dumps(
{
"dialogueId": dialogue.dialogue_id,
"startTurnIndex": dialogue.turns[0].turn_index,
"numTurns": num_total_turns,
"cheatingTurnIndices": cheating_turn_indices,
"numCheatingTurns": num_cheating_turns,
"pctCheatingTurns": num_cheating_turns / num_total_turns,
}
)
)
cheating_report_fp.write("\n")
complete_execution_results_fp.close()
cheating_report_fp.close()
cheating_stats_file = outbase + ".cheating_stats.json"
analyze_cheating_report(cheating_report_file, cheating_stats_file)
return complete_execution_results_file, cheating_report_file, cheating_stats_file
def main(
trade_data_file: str,
keep_all_domains: bool,
remove_none: bool,
fill_none: bool,
no_refer: bool,
no_revise: bool,
avoid_empty_plan: bool,
outbase: str,
) -> str:
utterance_tokenizer = UtteranceTokenizer()
salience_model: SalienceModelBase
if no_refer:
salience_model = DummySalienceModel()
else:
salience_model = VanillaSalienceModel()
total_num_refer_calls = 0
num_dialogues = 0
num_dialogues_without_refer_calls = 0
num_turns_with_refer: int = 0
num_turns: int = 0
trade_dialogues = json.load(open(trade_data_file, "r"))
dataflow_dialogues_file = outbase + ".dataflow_dialogues.jsonl"
dataflow_dialogues_jsonl_fp = open(dataflow_dialogues_file, "w")
refer_call_reports_jsonl_fp = open(outbase + ".refer_call_reports.jsonl",
"w")
refer_call_reports_txt_fp = open(outbase + ".refer_call_reports.txt", "w")
for trade_dialogue in tqdm(trade_dialogues, unit=" dialogues"):
(
dataflow_dialogue,
num_refer_calls,
refer_call_reports,
) = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=keep_all_domains,
remove_none=remove_none,
fill_none=fill_none,
salience_model=salience_model,
no_revise=no_revise,
avoid_empty_plan=avoid_empty_plan,
utterance_tokenizer=utterance_tokenizer,
)
dataflow_dialogues_jsonl_fp.write(
jsons.dumps(dataflow_dialogue) + "\n")
for refer_call_report in refer_call_reports:
if refer_call_report["numResolveCalls"] > 0:
num_turns_with_refer += 1
num_turns += 1
refer_call_reports_jsonl_fp.write(json.dumps(refer_call_report))
refer_call_reports_jsonl_fp.write("\n")
save_refer_call_report_txt(refer_call_report,
refer_call_reports_txt_fp)
total_num_refer_calls += num_refer_calls
num_dialogues += 1
if num_refer_calls == 0:
num_dialogues_without_refer_calls += 1
dataflow_dialogues_jsonl_fp.close()
refer_call_reports_jsonl_fp.close()
refer_call_reports_txt_fp.close()
# print some basic statistics
print(f"Converted {num_dialogues} dialogues")
print(
f"Number of dialogues without refer calls: {num_dialogues_without_refer_calls}"
)
print(f"Number of turns: {num_turns}")
print(f"Number of turns with refer calls: {num_turns_with_refer}")
print(f"Total number of refer calls: {total_num_refer_calls}")
return dataflow_dialogues_file
def test_execute_programs(trade_dialogue_1: Dict[str, Any]):
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
# ============================
# get cheating execution results
# ============================
dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue_1,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
complete_execution_results, cheating_turn_indices = execute_programs_for_dialogue(
dialogue=dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="never",
cheating_execution_results=None,
)
assert not cheating_turn_indices
for trade_turn, complete_execution_result in zip(
trade_dialogue_1["dialogue"], complete_execution_results):
assert build_belief_state_from_trade_turn(
trade_turn) == build_belief_state_from_belief_dict(
complete_execution_result.belief_dict, strict=True)
# pylint: disable=no-member
cheating_execution_results = {
turn.turn_index: complete_execution_result
for turn, complete_execution_result in zip(dataflow_dialogue.turns,
complete_execution_results)
}
# ============================
# mock the belief state predictions
# ============================
mock_belief_states = [
# turn 1: correct
{
"hotel-name": "none",
"hotel-type": "none"
},
# turn 2: correct
{
"hotel-name": "hilton",
"hotel-pricerange": "cheap",
"hotel-type": "guest house",
},
# turn 3: change a slot
{
"hotel-name": "none",
"hotel-pricerange": "_cheap",
"hotel-type": "guest house",
},
# turn 4: add a slot
{
"hotel-type": "_added"
},
# turn 5: correct
{
"hotel-area": "west"
},
# turn 6: correct
{
"hotel-area": "west",
"restaurant-area": "west"
},
# turn 7: correct
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
},
# turn 8: change two slots
{
"hotel-area": "_west",
"restaurant-area": "_west",
"restaurant-pricerange": "cheap",
},
# turn 9: correct
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
"taxi-departure": "none",
},
# turn 10: drop a slot
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
},
]
mock_trade_dialogue = build_trade_dialogue(
dialogue_id="mock",
turns=[("", "", belief_state) for belief_state in mock_belief_states],
)
mock_dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=mock_trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
_, mock_cheating_turn_indices = execute_programs_for_dialogue(
dialogue=mock_dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="always",
cheating_execution_results=cheating_execution_results,
)
assert mock_cheating_turn_indices == [
turn.turn_index for turn in dataflow_dialogue.turns
]
_, mock_cheating_turn_indices = execute_programs_for_dialogue(
dialogue=mock_dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="dynamic",
cheating_execution_results=cheating_execution_results,
)
assert mock_cheating_turn_indices == [3, 4, 8, 10]