def test_create_programs_with_vanilla_salience_model(data_dir: str):
"""Tests creating programs with a vanilla salience model.
"""
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
expected_num_refer_calls = {
"MUL1626.json": 1,
"PMUL3166.json": 0,
"MUL2258.json": 1,
"MUL2199.json": 1,
"MUL2096.json": 0,
"PMUL3470.json": 0,
"PMUL4478.json": 0,
}
for trade_dialogue in load_test_trade_dialogues(data_dir):
dataflow_dialogue, num_refer_calls, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
dialogue_id = dataflow_dialogue.dialogue_id
assert (num_refer_calls == expected_num_refer_calls[dialogue_id]
), "{} failed".format(dialogue_id)
def build_user_utterance(
text: str, utterance_tokenizer: UtteranceTokenizer) -> UserUtterance:
text = clean_utterance_text(text)
if not text:
return UserUtterance(original_text=SpecialStrings.NULL,
tokens=[SpecialStrings.NULL])
return UserUtterance(original_text=text,
tokens=utterance_tokenizer.tokenize(text))
def test_tokenize_utterance():
utterance_tokenizer = UtteranceTokenizer()
data = [
(
"Reschedule meeting with Barack Obama to 5/30/2019 at 3:00pm",
[
"Reschedule",
"meeting",
"with",
"Barack",
"Obama",
"to",
"5",
"/",
"30",
"/",
"2019",
"at",
"3",
":",
"00",
"pm",
],
),
(
"Can you also add icecream birthday tomorrow at 6PM?",
[
"Can",
"you",
"also",
"add",
"icecream",
"birthday",
"tomorrow",
"at",
"6",
"PM",
"?",
],
),
]
for text, expected in data:
assert utterance_tokenizer.tokenize(text) == expected
def build_agent_utterance(text: str, utterance_tokenizer: UtteranceTokenizer,
described_entities: List[str]) -> AgentUtterance:
text = clean_utterance_text(text)
if not text:
return AgentUtterance(
original_text=SpecialStrings.NULL,
tokens=[SpecialStrings.NULL],
described_entities=described_entities,
)
return AgentUtterance(
original_text=text,
tokens=utterance_tokenizer.tokenize(text),
described_entities=described_entities,
)
def test_create_programs_with_revise_with_fill_none(
trade_dialogue_1: Dict[str, Any]):
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
expected_plans: List[str] = [
# turn 1
"""(find (Constraint[Hotel] :area (?= #(String "none")) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-stay (?= #(String "none")) :internet (?= #(String "none")) :name (?= #(String "none")) :parking (?= #(String "none")) :pricerange (?= #(String "none")) :stars (?= #(String "none")) :type (?= #(String "none"))))""",
# turn 2
"""(ReviseConstraint :new (Constraint[Hotel] :name (?= #(String "hilton")) :pricerange (?= #(String "cheap")) :type (?= #(String "guest house"))) :oldLocation (Constraint[Constraint[Hotel]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 3
"""(ReviseConstraint :new (Constraint[Hotel] :name (?= #(String "none"))) :oldLocation (Constraint[Constraint[Hotel]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 4
"""(abandon (Constraint[Hotel]))""",
# turn 5
"""(find (Constraint[Hotel] :area (?= #(String "west")) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-stay (?= #(String "none")) :internet (?= #(String "none")) :name (?= #(String "none")) :parking (?= #(String "none")) :pricerange (?= #(String "none")) :stars (?= #(String "none")) :type (?= #(String "none"))))""",
# turn 6
"""(find (Constraint[Restaurant] :area (refer (Constraint[Area])) :book-day (?= #(String "none")) :book-people (?= #(String "none")) :book-time (?= #(String "none")) :food (?= #(String "none")) :name (?= #(String "none")) :pricerange (?= #(String "none"))))""",
# turn 7
"""(ReviseConstraint :new (Constraint[Restaurant] :pricerange (refer (Constraint[Pricerange]))) :oldLocation (Constraint[Constraint[Restaurant]]) :rootLocation (roleConstraint #(Path "output")))""",
# turn 8
"()",
# turn 9
"""(find (Constraint[Taxi] :arriveby (?= #(String "none")) :departure (?= #(String "none")) :destination (?= #(String "none")) :leaveat (?= #(String "none"))))""",
# turn 10
"()",
]
dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue_1,
keep_all_domains=True,
remove_none=False,
fill_none=True,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
for turn, expected_plan in zip(
dataflow_dialogue.turns,
expected_plans # pylint: disable=no-member
):
lispress = turn.lispress
assert lispress == expected_plan
def test_create_programs_without_revise(data_dir: str):
"""Tests creating programs without revise calls.
It should not use refer calls even with a valid salience model.
"""
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
for trade_dialogue in load_test_trade_dialogues(data_dir):
for avoid_empty_plan in [True, False]:
_, num_refer_calls, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=True,
avoid_empty_plan=avoid_empty_plan,
utterance_tokenizer=utterance_tokenizer,
)
assert num_refer_calls == 0
def main(
trade_data_file: str,
keep_all_domains: bool,
remove_none: bool,
fill_none: bool,
no_refer: bool,
no_revise: bool,
avoid_empty_plan: bool,
outbase: str,
) -> str:
utterance_tokenizer = UtteranceTokenizer()
salience_model: SalienceModelBase
if no_refer:
salience_model = DummySalienceModel()
else:
salience_model = VanillaSalienceModel()
total_num_refer_calls = 0
num_dialogues = 0
num_dialogues_without_refer_calls = 0
num_turns_with_refer: int = 0
num_turns: int = 0
trade_dialogues = json.load(open(trade_data_file, "r"))
dataflow_dialogues_file = outbase + ".dataflow_dialogues.jsonl"
dataflow_dialogues_jsonl_fp = open(dataflow_dialogues_file, "w")
refer_call_reports_jsonl_fp = open(outbase + ".refer_call_reports.jsonl",
"w")
refer_call_reports_txt_fp = open(outbase + ".refer_call_reports.txt", "w")
for trade_dialogue in tqdm(trade_dialogues, unit=" dialogues"):
(
dataflow_dialogue,
num_refer_calls,
refer_call_reports,
) = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue,
keep_all_domains=keep_all_domains,
remove_none=remove_none,
fill_none=fill_none,
salience_model=salience_model,
no_revise=no_revise,
avoid_empty_plan=avoid_empty_plan,
utterance_tokenizer=utterance_tokenizer,
)
dataflow_dialogues_jsonl_fp.write(
jsons.dumps(dataflow_dialogue) + "\n")
for refer_call_report in refer_call_reports:
if refer_call_report["numResolveCalls"] > 0:
num_turns_with_refer += 1
num_turns += 1
refer_call_reports_jsonl_fp.write(json.dumps(refer_call_report))
refer_call_reports_jsonl_fp.write("\n")
save_refer_call_report_txt(refer_call_report,
refer_call_reports_txt_fp)
total_num_refer_calls += num_refer_calls
num_dialogues += 1
if num_refer_calls == 0:
num_dialogues_without_refer_calls += 1
dataflow_dialogues_jsonl_fp.close()
refer_call_reports_jsonl_fp.close()
refer_call_reports_txt_fp.close()
# print some basic statistics
print(f"Converted {num_dialogues} dialogues")
print(
f"Number of dialogues without refer calls: {num_dialogues_without_refer_calls}"
)
print(f"Number of turns: {num_turns}")
print(f"Number of turns with refer calls: {num_turns_with_refer}")
print(f"Total number of refer calls: {total_num_refer_calls}")
return dataflow_dialogues_file
def test_execute_programs(trade_dialogue_1: Dict[str, Any]):
utterance_tokenizer = UtteranceTokenizer()
salience_model = VanillaSalienceModel()
# ============================
# get cheating execution results
# ============================
dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=trade_dialogue_1,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
complete_execution_results, cheating_turn_indices = execute_programs_for_dialogue(
dialogue=dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="never",
cheating_execution_results=None,
)
assert not cheating_turn_indices
for trade_turn, complete_execution_result in zip(
trade_dialogue_1["dialogue"], complete_execution_results):
assert build_belief_state_from_trade_turn(
trade_turn) == build_belief_state_from_belief_dict(
complete_execution_result.belief_dict, strict=True)
# pylint: disable=no-member
cheating_execution_results = {
turn.turn_index: complete_execution_result
for turn, complete_execution_result in zip(dataflow_dialogue.turns,
complete_execution_results)
}
# ============================
# mock the belief state predictions
# ============================
mock_belief_states = [
# turn 1: correct
{
"hotel-name": "none",
"hotel-type": "none"
},
# turn 2: correct
{
"hotel-name": "hilton",
"hotel-pricerange": "cheap",
"hotel-type": "guest house",
},
# turn 3: change a slot
{
"hotel-name": "none",
"hotel-pricerange": "_cheap",
"hotel-type": "guest house",
},
# turn 4: add a slot
{
"hotel-type": "_added"
},
# turn 5: correct
{
"hotel-area": "west"
},
# turn 6: correct
{
"hotel-area": "west",
"restaurant-area": "west"
},
# turn 7: correct
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
},
# turn 8: change two slots
{
"hotel-area": "_west",
"restaurant-area": "_west",
"restaurant-pricerange": "cheap",
},
# turn 9: correct
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
"taxi-departure": "none",
},
# turn 10: drop a slot
{
"hotel-area": "west",
"restaurant-area": "west",
"restaurant-pricerange": "cheap",
},
]
mock_trade_dialogue = build_trade_dialogue(
dialogue_id="mock",
turns=[("", "", belief_state) for belief_state in mock_belief_states],
)
mock_dataflow_dialogue, _, _ = create_programs_for_trade_dialogue(
trade_dialogue=mock_trade_dialogue,
keep_all_domains=True,
remove_none=False,
fill_none=False,
salience_model=salience_model,
no_revise=False,
avoid_empty_plan=False,
utterance_tokenizer=utterance_tokenizer,
)
_, mock_cheating_turn_indices = execute_programs_for_dialogue(
dialogue=mock_dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="always",
cheating_execution_results=cheating_execution_results,
)
assert mock_cheating_turn_indices == [
turn.turn_index for turn in dataflow_dialogue.turns
]
_, mock_cheating_turn_indices = execute_programs_for_dialogue(
dialogue=mock_dataflow_dialogue,
salience_model=salience_model,
no_revise=False,
cheating_mode="dynamic",
cheating_execution_results=cheating_execution_results,
)
assert mock_cheating_turn_indices == [3, 4, 8, 10]