def message_mutation(self, message: Message) -> Message:
if CONST.RETRIEVED_DOCS not in message:
return message
new_message = message.copy()
docs = message.get(CONST.RETRIEVED_DOCS)
new_docs = []
max_chunks = self.opt.get('woi_doc_max_chunks', 100)
keep = torch.randperm(len(docs))[0:max_chunks]
remove = torch.ones(len(docs))
remove[keep] = 0
for i in range(len(docs)):
if remove[i] == 0:
new_docs.append(docs[i])
else:
# We may still keep the doc if it contains the gold checked sentence(s).
checked_sentences = message.get(
CONST.SELECTED_SENTENCES,
message.get('labels', [CONST.NO_SELECTED_SENTENCES_TOKEN]),
)
d = docs[i]
found = False
if ' '.join(checked_sentences) != CONST.NO_SELECTED_SENTENCES_TOKEN:
for sent in checked_sentences:
s = sent.lstrip(' ').rstrip(' ')
if s in d:
found = True
if found:
new_docs.append(docs[i])
new_message.force_set(CONST.RETRIEVED_DOCS, new_docs)
return new_message
def act(self):
"""
Send new dialog message.
"""
if not hasattr(self, 'epochDone'):
# reset if haven't yet
self.reset()
# get next example, action is episode_done dict if already out of exs
action, self.epochDone = self.next_example()
# TODO: all teachers should eventually create messages
# while setting up the data, so this won't be necessary
action = Message(action)
action.force_set('id', self.getID())
# remember correct answer if available
self.lastY = action.get('labels_1', action.get('eval_labels_1', None))
if (not self.datatype.startswith('train')
or 'evalmode' in self.datatype) and 'labels' in action:
# move labels to eval field so not used for training
# but this way the model can use the labels for perplexity or loss
action = action.copy()
labels = action.pop('labels')
if not self.opt.get('hide_labels', False):
action['eval_labels'] = labels
return action
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
docs = message['text'].split('\n')
context = docs[-1]
docs = docs[0:-1]
new_message[CONST.RETRIEVED_DOCS] = docs
new_message.force_set('text', context)
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
new_message[CONST.RETRIEVED_DOCS] = [message['checked_sentence']]
new_message[CONST.RETRIEVED_DOCS_TITLES] = [message['title']]
new_message[CONST.RETRIEVED_DOCS_URLS] = ['']
new_message.pop('checked_sentence')
if message.get('history', '') != '':
text = message.get('history', '') + '\n' + message.get('text', '')
new_message.force_set('text', text)
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
if not isinstance(new_message[CONST.RETRIEVED_DOCS], list):
new_message.force_set(CONST.RETRIEVED_DOCS,
[new_message[CONST.RETRIEVED_DOCS]])
new_message[CONST.RETRIEVED_DOCS_TITLES] = [''] * len(
new_message[CONST.RETRIEVED_DOCS])
new_message[CONST.RETRIEVED_DOCS_URLS] = [''] * len(
new_message[CONST.RETRIEVED_DOCS])
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
new_message.pop('knowledge')
new_docs = [' '.join(message['knowledge'].split('\n'))]
new_titles = ['']
new_urls = ['']
new_message.force_set(CONST.RETRIEVED_DOCS, new_docs)
new_message.force_set(CONST.RETRIEVED_DOCS_TITLES, new_titles)
new_message.force_set(CONST.RETRIEVED_DOCS_URLS, new_urls)
return new_message
def test_message(self):
message = Message()
message['text'] = 'lol'
err = None
try:
message['text'] = 'rofl'
except RuntimeError as e:
err = e
assert err is not None, 'Message allowed override'
message_copy = message.copy()
assert type(message_copy) == Message, 'Message did not copy properly'
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
if 'text' not in message or 'labels' not in message or not message[
'labels']:
return message
labels = new_message.pop('labels')
checked_sentence = new_message.get(self.checked_sentence_kword, '')
if isinstance(checked_sentence, list):
checked_sentence = ' '.join(checked_sentence)
new_message['dialogue_response'] = labels
new_message['labels'] = [checked_sentence]
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
if 'text' not in message:
return message
text = new_message.pop('text')
checked_sentence = new_message.get(self.checked_sentence_kword, '')
if isinstance(checked_sentence, list):
checked_sentence = ' '.join(checked_sentence)
text += (
f'\n{CONST.KNOWLEDGE_TOKEN} {checked_sentence} {CONST.END_KNOWLEDGE_TOKEN}'
)
new_message['text'] = text
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
if 'text' not in message or 'labels' not in message or not message[
'labels']:
return message
if 'dialogue_response' in new_message:
# checked_sentence_as_label was applied before
labels = new_message['dialogue_response']
else:
labels = new_message['labels']
dialogue_response = labels[0]
text = new_message.pop('text')
text += f'\n{CONST.TOKEN_LABEL} {dialogue_response} {CONST.TOKEN_END_LABEL}'
new_message['text'] = text
return new_message
def message_mutation(self, message: Message) -> Message:
new_message = message.copy()
if 'text' not in message or 'labels' not in message or not message[
'labels']:
return message
if 'dialogue_response' in new_message:
# checked_sentence_as_label was applied before
labels = new_message['dialogue_response']
else:
labels = new_message['labels']
dialogue_response = labels[0]
text = new_message.pop('text')
ls = dialogue_response.split()
ind = random.randint(0, len(ls) - 1)
label1 = ' '.join(ls[0:ind])
label2 = ' '.join(ls[ind:len(ls)])
text += f'\n{label1}\n{CONST.TOKEN_LABEL} {label2} {CONST.TOKEN_END_LABEL}'
new_message['text'] = text
return new_message
def observe(self, observation: Message) -> Dict[str, Message]:
"""
Observe in 3 out of the 4 modules.
:param observation:
incoming message
:return self.observation:
returned observation is actually a dictionary mapping
agent module name to the corresponding observation
"""
if not isinstance(observation, Message):
observation = Message(observation)
for key in ['label_candidates', 'knowledge']:
# Delete unnecessarily large keys
observation.pop(key, '')
observation['knowledge_response'] = observation.get('checked_sentence', '')
raw_observation = copy.deepcopy(observation)
# This part is *specifically* for document chunking.
if self.krm_mutators:
observation = observation.copy()
for mutator in self.krm_mutators:
assert isinstance(mutator, MessageMutator), "not message mutator"
observation = next(mutator([observation]))
knowledge_observation = self.knowledge_agent.observe(observation)
knowledge_observation['prior_knowledge_responses'] = ' '.join(
self.knowledge_responses
)
if observation.get('episode_done'):
self.knowledge_responses = ['__SILENCE__']
search_query_observation = None
if self.search_query_agent:
sqm_obs = copy.deepcopy(observation)
if self.opt['search_query_control_token']:
sqm_obs.force_set(
'temp_history', f" {self.opt['search_query_control_token']}"
)
sqm_obs.force_set('skip_retrieval', True)
search_query_observation = self.search_query_agent.observe(sqm_obs)
search_decision_observation = None
if (
self.search_decision_agent
and self.search_decision is SearchDecision.COMPUTE
):
assert (
self.search_decision_agent.history.size == 1
), "wrong history size! set --sdm-history-size 1"
sdm_obs = copy.deepcopy(observation)
if self.opt['search_decision_control_token']:
sdm_obs.force_set(
'temp_history', f" {self.opt['search_decision_control_token']}"
)
sdm_obs.force_set('skip_retrieval', True)
search_decision_observation = self.search_decision_agent.observe(sdm_obs)
observations = {
'raw': raw_observation,
'knowledge_agent': knowledge_observation,
'search_query_agent': search_query_observation,
'search_decision_agent': search_decision_observation,
}
self.observations = observations
return observations
