def async_respond_to_queued_utts(session_key, user_utt, talker_names):
user_utt = U(user_utt)
failsafe_user_utt = {
'raw_utt': user_utt,
'spelled_utt': user_utt,
'spelled_tags': [],
'corefed_utt': user_utt,
'corefed_tags': []
}
failsafe_bot_response = {
'talker_name': "failsafe",
'utt': u'Sorry, could you say that again, please :)',
'score': -1.0,
'confidence': -1.0,
'talker_weight': 1.0
}
pipe = db.pipeline()
pipe.set(last_bot_utt_key(session_key), AsyncBot.none_pickle, nx=True)
pipe.set(new_user_utt_key(session_key), AsyncBot.none_pickle, nx=True)
pipe.rename(new_user_utt_key(session_key),
last_user_utt_key(session_key))
pipe.set(new_user_utt_key(session_key),
pickle.dumps(failsafe_user_utt, -1))
pipe.delete(new_bot_utts_key(session_key))
pipe.zadd(new_bot_utts_key(session_key),
pickle.dumps(failsafe_bot_response, -1),
-failsafe_bot_response['score'])
pipe.delete(new_bot_followups_key(session_key))
pipe.execute()
responders = [
async_talkers[tn] for tn in talker_names if
not getattr(async_talkers[tn].klass._respond_to, '_no_op', False)
]
follow_uppers = [
async_talkers[tn] for tn in talker_names
if not getattr(async_talkers[tn].klass.follow_up, '_no_op', False)
]
sa = dict(immutable=True)
if config.celery_timeouts:
sa['soft_time_limit'] = config.talker_respond_timeout
sa['time_limit'] = config.talker_respond_timeout + 2
job = (
AsyncBot.preprocessor.preprocess.signature(
(user_utt, session_key), **sa) | celery.group(
t.respond_to.signature((session_key, ), **sa)
for t in responders) |
# add noop because of https://github.com/celery/celery/issues/3585
AsyncBot.noop.si(session_key))
if follow_uppers:
job = (job | celery.group(
t.follow_up.signature((session_key, ), **sa)
for t in follow_uppers))
job = job | AsyncBot.combine_responses_and_follow_ups.signature(
(session_key, ), **sa)
return job.delay(time_limit=3)
def preprocess(user_raw_utt, session_key):
try:
pipe = db.pipeline(transaction=False)
pipe.get(last_bot_utt_key(session_key))
state, pret = self.get_state(session_key, pipe)
last_bot_utt = pickle.loads(pret[0])
state, user_utt_dict = self.object.preprocess(
state, user_raw_utt, last_bot_utt)
pipe = db.pipeline(transaction=False)
pipe.set(new_user_utt_key(session_key),
pickle.dumps(user_utt_dict, -1))
self.save_state(session_key, state, pipe)
except:
handle_async_error(session_key=session_key)
def set_article(article, session_key):
try:
state = self.get_state(session_key)
state, article_dict = self.object.set_article(state, article)
pipe = db.pipeline(transaction=False)
pipe.set(article_key(session_key),
pickle.dumps(article_dict, -1))
self.save_state(session_key, state, pipe)
except:
handle_async_error(session_key=session_key)
def set_article(session_key):
try:
if getattr(self.object.set_article, '_no_op', False):
return
pipe = db.pipeline(transaction=False)
pipe.get(article_key(session_key))
state, pret = self.get_state(session_key, pipe)
article = pickle.loads(pret[0])
state = self.object.set_article(state, article)
self.save_state(session_key, state)
except:
handle_async_error(session_key=session_key)
def follow_up(session_key):
try:
if getattr(self.object.follow_up, '_no_op', False):
return
pipe = db.pipeline(transaction=False)
pipe.zrange(new_bot_utts_key(session_key), 0, -1)
state, pret = self.get_state(session_key, pipe)
new_bot_utts = [pickle.loads(bu) for bu in pret[0]]
state, new_bot_utt, confidence = self.object.follow_up(
state, new_bot_utts)
pipe = db.pipeline(transaction=False)
if confidence is not None and confidence > 0:
fu_data = pickle.dumps(
(self.klass.__name__, new_bot_utt, confidence), -1)
pipe.zadd(new_bot_followups_key(session_key), fu_data,
-confidence)
self.save_state(session_key, state, pipe)
except:
handle_async_error(session_key=session_key)
def respond_to(session_key):
try:
if getattr(self.object._respond_to, '_no_op', False):
return
pipe = db.pipeline(transaction=False)
pipe.get(last_user_utt_key(session_key))
pipe.get(last_bot_utt_key(session_key))
pipe.get(new_user_utt_key(session_key))
state, pipe_ret = self.get_state(session_key, pipe)
(last_user_utt_dict, last_bot_utt,
user_utt_dict) = (pickle.loads(r) for r in pipe_ret)
state, bot_utt, confidence = self.object.respond_to(
state, last_user_utt_dict, last_bot_utt, user_utt_dict)
if confidence is None or bot_utt is None:
self.save_state(session_key, state)
return
name = self.klass.__name__
weight = AsyncBot.talker_weight[name]
score = confidence * weight
utts = [{
'talker_name': name,
'utt': chatbot.postprocess_utt(bot_utt),
'score': score,
'confidence': confidence,
'talker_weight': weight
}]
if self.klass.apply_profanity:
chatbot.filter_nsfw_utterances(utts, user_utt_dict)
pipe = db.pipeline(transaction=False)
pipe.zadd(new_bot_utts_key(session_key),
pickle.dumps(utts[0], -1), -utts[0]['score'])
self.save_state(session_key, state, pipe)
except:
handle_async_error(session_key=session_key)
def get_state(self, session_key, pipe=None):
_pipe = pipe or db.pipeline()
state_key = session_key + self.state_suffix
_pipe.get(state_key)
pret = _pipe.execute()
state_pickle = pret.pop(-1)
if state_pickle is None:
print('Computning new state for: %s' % (self.base_name, ))
state = self.object.new_state()
else:
state = pickle.loads(state_pickle)
if pipe is not None:
return state, pret
else:
return state
def combine_responses_and_follow_ups(session_key):
pipe = db.pipeline()
pipe.zrange(new_bot_utts_key(session_key), 0, -1)
pipe.delete(new_bot_utts_key(session_key))
pipe.zrange(new_bot_followups_key(session_key), 0, -1)
pipe.delete(new_bot_followups_key(session_key))
utts, _, follow_ups, _ = pipe.execute()
utts = [pickle.loads(u) for u in utts]
follow_ups = [pickle.loads(fu) for fu in follow_ups]
new_bot_utt = utts[0]['utt']
utt_table = []
sel_fu = None
if follow_ups:
sel_fu = chatbot.select_follow_up(follow_ups)
if sel_fu:
name, fu, confidence = sel_fu
new_bot_utt = chatbot.postprocess_utt(fu)
utt_table.append({
'talker_name': name + ' sel_fup',
'utt': fu,
'score': confidence,
'confidence': confidence,
'talker_weight': 1.0
})
for name, fu, confidence in follow_ups:
score = confidence
if confidence and 1 < confidence:
score = 1.0 + (score - 1.0) * config.talker_weight[name]
utt_table.append({
'talker_name': name + ' fup',
'utt': fu,
'score': score,
'confidence': confidence,
'talker_weight': 1.0
})
utt_table.extend(utts)
db.set(last_bot_utt_key(session_key), pickle.dumps(new_bot_utt, -1))
return new_bot_utt, utt_table