def test_get_entity_by_wiki_name(self):
"""Test the get_entity_by_wiki_name function"""
entity = get_entity_by_wiki_name('Chicago (2002 film)')
self.assertEqual(entity.doc_id, 201534) # Chicago the movie
entity = get_entity_by_wiki_name('Chicago (2002 filmm)')
self.assertIsNone(entity) # there is no wikipedia article by this name
def get_response(self, state: State,
state_manager) -> ResponseGeneratorResult:
"""Ask the first unasked question for state.cur_category_name"""
category_name = state.cur_category_name
question = state.get_first_category_response(
category_name, state_manager) # CategoryQuestion or None
if question:
question_str = None
if question.statement is None:
question_str = question.question
elif question.question is None:
question_str = question.statement
else:
question_str = ' '.join(
(question.statement, question.question))
response = "{} {}".format(choice(ACKNOWLEDGEMENTS), question_str)
priority = ResponsePriority.CAN_START if category_name == HistoryCategory.__name__ else ResponsePriority.FORCE_START
cur_entity = get_entity_by_wiki_name(question.cur_entity_wiki_name,
state_manager.current_state)
conditional_state = ConditionalState(HandleAnswerTreelet.__name__,
category_name,
question.statement,
question.question, False)
return ResponseGeneratorResult(
text=response,
priority=priority,
needs_prompt=False,
state=state,
cur_entity=cur_entity,
expected_type=question.expected_type,
conditional_state=conditional_state)
else:
return emptyResult(state)
def get_response(self, state: dict) -> ResponseGeneratorResult:
utterance = self.state_manager.current_state.text.lower()
nav_intent_output = self.state_manager.current_state.navigational_intent
if self.talk_about_george_floyd(state, utterance):
blm_entity = get_entity_by_wiki_name("Black Lives Matter")
return ResponseGeneratorResult(text=RESPONSE_TO_QUESTION_ONE_GEORGE_FLOYD,
priority=ResponsePriority.FORCE_START,
needs_prompt=True, state=state,
cur_entity=blm_entity, conditional_state={"talked_about_blm": True},
smooth_handoff=SmoothHandoff.ONE_TURN_TO_WIKI_GF)
# Check for chatty phrases in utterance
slots = ChattyTemplate().execute(utterance)
my_name_slots = MyNameIsNonContextualTemplate().execute(utterance)
not_my_name_slots = MyNameIsNotTemplate().execute(utterance)
if slots is not None:
chatty_phrase = slots["chatty_phrase"]
logger.primary_info('Detected chatty phrase intent with slots={}'.format(slots))
# Step 3: Get response from dictionary of hand-written responses
response, needs_prompt = one_turn_responses[chatty_phrase]
logger.primary_info('Chatty RG returned user_response={}'.format(response))
# Check for user hesitating while trying to navigate to a topic
elif nav_intent_output.pos_intent and nav_intent_output.pos_topic_is_hesitate and "depends on" not in utterance:
logger.primary_info('User has PositiveNavigationalIntent with topic=HESITATE, so asking them for topic again')
response, needs_prompt = "I think I missed the last part of that sentence. Can you tell me one more time what you want to talk about?", False
# Check for user giving general positive talking intent (e.g. "i want to chat")
# If WIKI is supposed to handle the utterance and it contains tell, it typically means user is asking for more info (and hence doesn't really specify topic)
elif nav_intent_output.pos_intent and nav_intent_output.pos_topic is None and not (self.state_manager.last_state_active_rg == 'WIKI' and contains_phrase(utterance, {'tell'})):
logger.primary_info('User has PositiveNavigationalIntent with topic=None, so ONE_TURN_HACK is responding with "What would you like to talk about?"')
response, needs_prompt = "Ok, I'd love to talk to you! What would you like to talk about?", False
# Check for user correcting their name
elif (my_name_slots and self.state_manager.last_state_active_rg and not self.state_manager.last_state_active_rg == 'LAUNCH') or not_my_name_slots:
logger.primary_info('User is attempting to correct name.')
response = "Oops, it sounds like I got your name wrong. I'm so sorry about that! I won't make that mistake again."
needs_prompt = True
setattr(self.state_manager.user_attributes, 'name', None)
# Otherwise return empty
else:
return emptyResult(state)
# Step 7: set priority
priority = ResponsePriority.FORCE_START
is_safe = True
# Step 8: return result
return ResponseGeneratorResult(text=response, priority=priority, needs_prompt=needs_prompt, state=state,
cur_entity=None, conditional_state=state)
def introduce_entity(self, state: State, is_blm=False):
latest_untalked_entity = None
text = None
if is_blm:
latest_untalked_entity = get_entity_by_wiki_name("Black Lives Matter")
text = "I am proud to support the Black Lives Matter movement. Would you like to learn more about Black Lives Matter?"
else:
latest_untalked_entity = self.rg.get_untalked_entity(state)
text = self.rg.state_manager.current_state.choose_least_repetitive(ENTITY_INTRODUCTION_TEMPLATES).format(entity_name=latest_untalked_entity.common_name)
conditional_state = ConditionalState(
cur_doc_title=latest_untalked_entity.name,
responding_treelet=self.__repr__(),
prompt_handler=self.__repr__())
return latest_untalked_entity, text, conditional_state
def get_prompt(self, state: State) -> PromptResult:
"""This method is the same as get_response. Only difference is that
1. we do not get a list of reasons (because there is not enough time)
2. we only chose to solicit an opinion if user didn't have an opinion on it before, or
3. we will solicit a reason if user told us before that they like or dislike a phrase
:param state: the current state
:type state: State
:return: a result chirpy can use
:rtype: PromptResult
"""
self.initialize_turn()
if state.last_turn_prompt:
return emptyPrompt(state)
phrase, priority, transition_phrase = self.select_phrase_for_prompt(state)
if phrase is None:
return emptyPrompt(state)
utterance = self.state_manager.current_state.text
additional_features = self.populate_features(state, utterance)
additional_features.detected_phrases = tuple([phrase for phrase in additional_features.detected_phrases if phrase not in state.phrases_done])
# Then need to advance the state using the utterance
state_p = state_actions.next_state(state, utterance, additional_features)
if state_p is None:
# Special handling for prompt: Add phrase and sentiment
state_p = state.reset_state()
state_p.cur_phrase = phrase
user_sentiments_history = dict(state_p.user_sentiment_history)
state_p.cur_sentiment = user_sentiments_history[phrase] if phrase in user_sentiments_history else 2
if state_p.cur_sentiment != 2:
action = Action(solicit_reason=True)
else:
action = Action(solicit_opinion=True)
# Then need to utterancify the action
text, phrase, reason = fancy_utterancify_prompt(state_p, action, [], [], [], state_p.cur_phrase not in state.detected_opinionated_phrases,
self.state_manager.current_state.choose_least_repetitive) # type: ignore
# Then need to fill the rest of the fields of state_p (through mutation)
state_p = state_actions.fill_state_on_action(state_p, action, text, phrase, additional_features, reason,
self.opinionable_phrases, self.opinionable_entities)
state_p.last_turn_prompt, state_p.last_turn_select = True, False
wiki_entity = None
if self.opinionable_phrases[phrase].good_for_wiki:
wiki_entity = get_entity_by_wiki_name(self.opinionable_phrases[phrase].wiki_entity_name)
state.last_turn_prompt, state.last_turn_select = False, False
text = ' '.join((transition_phrase, text))
return PromptResult(text, priority, state, wiki_entity, conditional_state=state_p)
def get_prompt(self, state: State, state_manager) -> PromptResult:
"""Ask the first unasked question for state.cur_category_name"""
category_name = state.cur_category_name
question = state.get_first_unasked_question(
category_name) # CategoryQuestion or None
if question:
question_str = question.question
response = "{} {}".format(choice(BRIDGES), question_str)
cur_entity = get_entity_by_wiki_name(question.cur_entity_wiki_name,
state_manager.current_state)
conditional_state = ConditionalState(HandleAnswerTreelet.__name__,
category_name, None,
question_str, False)
return PromptResult(text=response,
prompt_type=PromptType.CURRENT_TOPIC,
state=state,
cur_entity=cur_entity,
expected_type=question.expected_type,
conditional_state=conditional_state)
else:
return emptyPrompt(state)
def get_by_name(cls, name):
try:
wiki_entity = get_entity_by_wiki_name(name)
except Exception:
wiki_entity = None
return wiki_entity
def get_prompt(self, state: State) -> PromptResult:
"""
Randomly choose an undiscussed generic category and ask its first question.
"""
# If this is a smooth transition from movies, get a prompt for tv category from the Introductory treelet
'''
if self.state_manager.current_state.smooth_handoff == SmoothHandoff.MOVIES_TO_CATEGORIES:
logger.primary_info('Getting prompt from tv IntroductoryTreelet.')
state.cur_category_name = 'TVCategory'
question = state.get_first_unasked_question(state.cur_category_name)
if question is None:
logger.warning('Unable to complete smooth handoff from movies to categories because we have no unasked tv questions left')
else:
new_entity = get_entity_by_wiki_name(question.cur_entity_wiki_name, self.state_manager.current_state)
conditional_state = ConditionalState(HandleAnswerTreelet.__name__, state.cur_category_name, None, question.question, False)
result = PromptResult(question.question, PromptType.FORCE_START, state, cur_entity=new_entity,
expected_type=question.expected_type, conditional_state=conditional_state)
return result
'''
# If the user is requesting a category, get a prompt for that category from the Introductory treelet
utterance = self.state_manager.current_state.text.lower()
user_initiated_category, user_has_posnav = get_user_initiated_category(
utterance, self.state_manager.current_state)
if user_initiated_category is not None:
state.cur_category_name = user_initiated_category
logger.primary_info(
f'Getting prompt from {IntroductoryTreelet.__name__}')
prompt_result = self.treeletname2treelet[
IntroductoryTreelet.__name__].get_prompt(
state, self.state_manager)
if prompt_result.text is not None and not user_has_posnav:
logger.primary_info(
"Setting prompt type to CONTEXTUAL as the user does not have posnav"
)
prompt_result.type = PromptType.CONTEXTUAL
if prompt_result:
return prompt_result
# Ask any remaining unasked generic category questions
category_name, question = state.get_random_generic_undiscussed_category_question(
)
if category_name and question:
logger.primary_info(
f"Randomly chose an undiscussed generic category {category_name}. Asking its first question as a generic prompt."
)
new_entity = get_entity_by_wiki_name(
question.cur_entity_wiki_name,
self.state_manager.current_state)
conditional_state = ConditionalState(HandleAnswerTreelet.__name__,
category_name, None,
question.question, False)
transition = self.state_manager.current_state.choose_least_repetitive(
QUESTION_CONNECTORS)
result = PromptResult(transition + question.question,
PromptType.GENERIC,
state,
cur_entity=new_entity,
expected_type=question.expected_type,
conditional_state=conditional_state)
return result
# Otherwise return nothing
return emptyPrompt(state)
def get_response(self, state : State) -> ResponseGeneratorResult:
"""This function defines the stages that we go through to generate the result. The procedure is
1. First populate the "additional_features"
2. Incorporate unconditional information such as user's likes and dislikes, phrases that were detected
3. Advance the state to the next state depending on the user's utterance and additional features
4. Define the action space for the policy
5. Select an action using a policy
6. Utterancify the action chosen using additional information like lists of reasons and alternatives
7. Post process the state conditioned on the action
:param state: the current state
:type state: State
:return: a result that can be used for chirpy
:rtype: ResponseGeneratorResult
"""
self.initialize_turn()
neg_intent = self.state_manager.current_state.navigational_intent.neg_intent # type: ignore
if neg_intent and self.state_manager.last_state_active_rg == 'OPINION': # type: ignore
return self.respond_neg_nav(state, None)
utterance = self.state_manager.current_state.text
additional_features = self.populate_features(state, utterance)
high_prec = self.state_manager.current_state.entity_linker.high_prec # type: ignore
# should_evaluate = len(state.action_history) > 4 and not state.evaluated \
# and not state.first_episode and state.cur_policy != '' and state.cur_policy != repr(OneTurnAgreePolicy())
should_evaluate = False # Turning off evaluation question.
if self.state_manager.current_state.entity_linker.high_prec and state.cur_phrase != '': # type: ignore
cur_entity_name = self.opinionable_phrases[state.cur_phrase].wiki_entity_name
if (cur_entity_name is None and state.cur_phrase not in [linked_span.span for linked_span in high_prec]) \
or (cur_entity_name is not None and cur_entity_name not in [linked_span.top_ent.name for linked_span in high_prec]):
# If the above condition passes, it means that the linked entity is not the currently opinionating phrase.
if len(additional_features.detected_phrases) == 0:
# User no longer want to talk about an opinionable phrase
return self.respond_neg_nav(state, random.choice(self.state_manager.current_state.entity_linker.high_prec).top_ent) # type: ignore
if state.last_turn_prompt or state.last_turn_select:
priority = ResponsePriority.STRONG_CONTINUE
elif len(high_prec) > 0 and \
not any(linked_span.span in self.opinionable_phrases \
or linked_span.top_ent.name in self.opinionable_entities for linked_span in high_prec): # type: ignore
self.logger.primary_info(f'Opinion realized that there is a high precision entity, will not CAN_START our conversation') # type: ignore
priority = ResponsePriority.NO
# if WhatsYourOpinion().execute(utterance) is not None:
# self.logger.primary_info(f"Opinion detected user is asking for our opinion, raising priority to FORCE_START") # type: ignore
# priority = ResponsePriority.FORCE_START
else:
priority = ResponsePriority.CAN_START
if len(state.action_history) > 0 and state.action_history[-1].exit:
self.logger.primary_info(f'Opinion detected our previous action is to exit and we were not selected, will reset the state before this turn starts') # type: ignore
state = state.reset_state()
priority = ResponsePriority.CAN_START # Drop the priority to CAN_START because we already ended a convo before
# First need to incorporate the unconditional information learned from this turn
state.detected_opinionated_phrases += additional_features.detected_phrases
if len(additional_features.detected_phrases) > 0:
# Here we only use regex since sentiment analysis may not always do well
if utils.is_like(utterance)[0]:
state.user_sentiment_history += tuple((phrase, 4) for phrase in additional_features.detected_phrases)
elif utils.is_not_like(utterance)[0]:
state.user_sentiment_history += tuple((phrase, 0) for phrase in additional_features.detected_phrases)
additional_features.detected_phrases = tuple([phrase for phrase in additional_features.detected_phrases if phrase not in state.phrases_done])
# Then need to advance the state using the utterance
state_p = state_actions.next_state(state, utterance, additional_features)
if state_p is None or state_p.cur_phrase is None:
return emptyResult(state.reset_state())
reasons_used = dict(state.reasons_used)
phrase_reasons_used = set(reasons_used[state_p.cur_phrase]) if state_p.cur_phrase in reasons_used else []
pos_reasons, neg_reasons = utils.get_reasons(state_p.cur_phrase)
pos_reasons = [reason for reason in pos_reasons if reason not in phrase_reasons_used]
neg_reasons = [reason for reason in neg_reasons if reason not in phrase_reasons_used]
related_entities = [phrase.text for phrase in self.opinionable_phrases.values() \
if phrase.category is not None and phrase.category == self.opinionable_phrases[state_p.cur_phrase].category \
and phrase.wiki_entity_name != self.opinionable_phrases[state_p.cur_phrase].wiki_entity_name]
related_entities = [e for e in related_entities if e not in state_p.phrases_done]
# Then need to define the action space
action_space = self.get_action_space(state_p, pos_reasons, neg_reasons, related_entities)
# Then need to select a policy if we don't have one
ab_test_policy = self.state_manager.current_state.experiments.look_up_experiment_value('opinion_policy') # type: ignore
if state_p.cur_policy != '':
self.logger.primary_info(f'OPINION is using current policy {state_p.cur_policy} to respond to the user') # type: ignore
elif ab_test_policy != 'random' or ab_test_policy == 'not_defined':
state_p.cur_policy = ab_test_policy
self.logger.primary_info(f'Opinion detected a/b test policy is {ab_test_policy}, will set current episode accordingly ') # type: ignore
elif state_p.num_turns_since_long_policy < 20:
policies, weights = zip(*self.short_policy_rates)
state_p.cur_policy = random.choices(policies, weights, k=1)[0] # type: ignore
self.logger.primary_info(f'Opinion had a long conversation {state_p.num_turns_since_long_policy} < 20 turns ago. Will use policy {state_p.cur_policy}') # type: ignore
else:
if state_p.last_policy in set([p for p, _ in self.disagree_policy_rates]):
policies, weights = zip(*self.agree_policies_rates)
elif state_p.last_policy in set([p for p, _ in self.agree_policies_rates]):
policies, weights = zip(*self.agree_policies_rates)
else:
policies, weights = zip(*self.policy_rates)
state_p.cur_policy = random.choices(policies, weights, k=1)[0] # type: ignore
self.logger.primary_info(f'OPINION have no current policy, randomly picked {state_p.cur_policy} to respond to the user, resetting turn count') # type: ignore
state_p.last_policy = state_p.cur_policy
policy = self.policies[state_p.cur_policy] # type: ignore
# Then need to get the action from a policy
action = policy.get_action(state_p, action_space, additional_features)
self.logger.primary_info(f'OPINION\'s strategy chose action {action}') # type: ignore
action_space = self.get_action_space(state_p, pos_reasons, neg_reasons, related_entities) # Redefine action space for checks in case cur_phrase changed
if action not in action_space:
self.logger.error(f'OPINION policy {repr(policy)} generated an action {action} that is not in the action space {action_space}. Check policy implementation.')
new_state = state.reset_state()
return emptyResult(new_state)
# Then need to utterancify the action
text, phrase, reason = fancy_utterancify(state_p, action, pos_reasons, neg_reasons,
related_entities, should_evaluate, self.state_manager.current_state.choose_least_repetitive) # type: ignore
# Then need to fill the rest of the fields of state_p (through mutation)
state_p = state_actions.fill_state_on_action(state_p, action, text, phrase, additional_features, reason,
self.opinionable_phrases, self.opinionable_entities)
state_p.last_turn_select = True
state_p.last_turn_prompt = False
user_sentiment_history_dict = dict(state.user_sentiment_history)
wiki_entity = None
if phrase != '' and phrase in user_sentiment_history_dict and user_sentiment_history_dict[phrase] > 2 \
and self.opinionable_phrases[phrase].good_for_wiki:
wiki_entity = get_entity_by_wiki_name(self.opinionable_phrases[phrase].wiki_entity_name)
state.last_turn_prompt, state.last_turn_select = False, False
needs_prompt = False
if action.exit:
if len(state_p.action_history) > 6:
self.logger.primary_info(f"Opinion had a conversation of length {len(state_p.action_history)}, will reset long_policy count") # type: ignore
state_p.num_turns_since_long_policy = 0
if not should_evaluate:
needs_prompt = True
if len(state_p.action_history) < 4:
self.logger.primary_info(f"Opinion only had 4 turns. Will WEAK_CONTINUE the conversation") # type: ignore
priority = ResponsePriority.WEAK_CONTINUE
state_p.first_episode = False
return ResponseGeneratorResult(text, priority, needs_prompt, state, wiki_entity, conditional_state=state_p)
user=user,
password=password)
cur = conn.cursor()
cur.execute(sql_statement)
conn.commit()
cur.close()
return
def get_ent_group(entity):
for ent_group_name, ent_group in ENTITY_GROUPS_FOR_CLASSIFICATION.ordered_items:
if ent_group.matches(entity):
return ent_group_name
return None
if __name__ == "__main__":
results = fetch_sql(
"select * from labeled_phrases_cat where generic = true")
id_to_wiki_cat = {
row[0]: get_ent_group(get_entity_by_wiki_name(row[3]))
if row[3] is not None else None
for row in results
}
for phrase_id, wiki_cat in id_to_wiki_cat.items():
if wiki_cat is not None:
execute_sql(
f"update labeled_phrases_cat set wiki_category = '{wiki_cat}' where id = {phrase_id}"
)
