class AIKernal(object): def __init__(self, load_all_modules=False): self.config = misc.load_config('.airc') # # database # Session = sessionmaker(bind=model.engine) self.session = Session() # # TensorFlow (deferred, as tf can take quite a bit of time to set up) # self.tf_session = None self.nlp_model = None # # module management, setup # self.modules = {} self.initialized_modules = set() s = self.config.get('semantics', 'modules') self.all_modules = list(map(lambda s: s.strip(), s.split(','))) sys.path.append('modules') # # AIProlog parser, runtime # db_url = self.config.get('db', 'url') self.db = LogicDB(db_url) self.aip_parser = AIPrologParser(self) self.rt = AIPrologRuntime(self.db) self.dummyloc = SourceLocation('<rt>') # # alignment / word2vec (on-demand model loading) # self.w2v_model = None self.w2v_lang = None self.w2v_all_utterances = [] # # load modules, if requested # if load_all_modules: for mn2 in self.all_modules: self.load_module(mn2) self.init_module(mn2) # FIXME: this will work only on the first call def setup_tf_model(self, mode, load_model, ini_fn, global_step=0): if not self.tf_session: import tensorflow as tf # setup config to use BFC allocator config = tf.ConfigProto() # config.gpu_options.allocator_type = 'BFC' self.tf_session = tf.Session(config=config) if not self.nlp_model: from nlp_model import NLPModel self.nlp_model = NLPModel(self.session, ini_fn, global_step=global_step) if load_model: self.nlp_model.load_dicts() # we need the inverse dict to reconstruct the output from tensor self.inv_output_dict = { v: k for k, v in viewitems(self.nlp_model.output_dict) } self.tf_model = self.nlp_model.create_tf_model(self.tf_session, mode=mode) self.tf_model.batch_size = 1 self.tf_model.restore(self.tf_session, self.nlp_model.model_fn) def clean(self, module_names, clean_all, clean_logic, clean_discourses, clean_cronjobs): for module_name in module_names: if clean_logic or clean_all: logging.info('cleaning logic for %s...' % module_name) if module_name == 'all': self.db.clear_all_modules() else: self.db.clear_module(module_name) if clean_discourses or clean_all: logging.info('cleaning discourses for %s...' % module_name) if module_name == 'all': self.session.query(model.DiscourseRound).delete() else: self.session.query(model.DiscourseRound).filter( model.DiscourseRound.module == module_name).delete() if clean_cronjobs or clean_all: logging.info('cleaning cronjobs for %s...' % module_name) if module_name == 'all': self.session.query(model.Cronjob).delete() else: self.session.query(model.Cronjob).filter( model.Cronjob.module == module_name).delete() self.session.commit() def load_module(self, module_name): if module_name in self.modules: return self.modules[module_name] logging.debug("loading module '%s'" % module_name) # fp, pathname, description = imp.find_module(module_name, ['modules']) fp, pathname, description = imp.find_module(module_name) # print fp, pathname, description m = None try: m = imp.load_module(module_name, fp, pathname, description) self.modules[module_name] = m # print m # print getattr(m, '__all__', None) # for name in dir(m): # print name for m2 in getattr(m, 'DEPENDS'): self.load_module(m2) if hasattr(m, 'CRONJOBS'): # update cronjobs in db old_cronjobs = set() for cronjob in self.session.query(model.Cronjob).filter( model.Cronjob.module == module_name): old_cronjobs.add(cronjob.name) new_cronjobs = set() for name, interval, f in getattr(m, 'CRONJOBS'): logging.debug('registering cronjob %s' % name) cj = self.session.query(model.Cronjob).filter( model.Cronjob.module == module_name, model.Cronjob.name == name).first() if not cj: cj = model.Cronjob(module=module_name, name=name, last_run=0) self.session.add(cj) cj.interval = interval new_cronjobs.add(cj.name) for cjn in old_cronjobs: if cjn in new_cronjobs: continue self.session.query(model.Cronjob).filter( model.Cronjob.module == module_name, model.Cronjob.name == cjn).delete() self.session.commit() if hasattr(m, 'init_module'): initializer = getattr(m, 'init_module') initializer(self) except: logging.error('failed to load module "%s"' % module_name) logging.error(traceback.format_exc()) sys.exit(1) finally: # Since we may exit via an exception, close fp explicitly. if fp: fp.close() return m def init_module(self, module_name, run_trace=False): if module_name in self.initialized_modules: return logging.debug("initializing module '%s'" % module_name) self.initialized_modules.add(module_name) m = self.load_module(module_name) if not m: raise Exception('init_module: module "%s" not found.' % module_name) for m2 in getattr(m, 'DEPENDS'): self.init_module(m2) prolog_s = u'init(\'%s\')' % (module_name) c = self.aip_parser.parse_line_clause_body(prolog_s) self.rt.set_trace(run_trace) solutions = self.rt.search(c) def compile_module(self, module_name): m = self.modules[module_name] # clear module, delete old NLP training data self.db.clear_module(module_name, commit=True) self.session.query(model.TrainingData).filter( model.TrainingData.module == module_name).delete() self.session.query(model.TestCase).filter( model.TestCase.module == module_name).delete() self.session.query(model.NERData).filter( model.NERData.module == module_name).delete() # extract new training data for this module train_ds = [] tests = [] ner = {} if hasattr(m, 'nlp_train'): # training_data_cnt = 0 logging.info('module %s python training data extraction...' % module_name) nlp_train = getattr(m, 'nlp_train') train_ds.extend(nlp_train(self)) if hasattr(m, 'nlp_test'): logging.info('module %s python test case extraction...' % module_name) nlp_test = getattr(m, 'nlp_test') nlp_tests = nlp_test(self) tests.extend(nlp_tests) if hasattr(m, 'AIP_SOURCES'): logging.info('module %s AIP training data extraction...' % module_name) for inputfn in m.AIP_SOURCES: ds, ts, ne = self.aip_parser.compile_file( 'modules/%s/%s' % (module_name, inputfn), module_name) train_ds.extend(ds) tests.extend(ts) for lang in ne: if not lang in ner: ner[lang] = {} for cls in ne[lang]: if not cls in ner[lang]: ner[lang][cls] = {} for entity in ne[lang][cls]: ner[lang][cls][entity] = ne[lang][cls][entity] logging.info( 'module %s training data extraction done. %d training samples, %d tests' % (module_name, len(train_ds), len(tests))) # put training data into our DB td_set = set() td_list = [] for utt_lang, contexts, i, resp, loc_fn, loc_line, loc_col, prio in train_ds: inp = copy(contexts) inp.extend(i) inp_json = json.dumps(inp) resp_json = json.dumps(resp) # utterance = u' '.join(map(lambda c: text_type(c), contexts)) # if utterance: # utterance += u' ' # utterance += u' '.join(i) utterance = u' '.join(i) k = utt_lang + '#0#' + '#' + inp_json + '#' + resp_json if not k in td_set: td_set.add(k) td_list.append( model.TrainingData( lang=utt_lang, module=module_name, utterance=utterance, inp=inp_json, resp=resp_json, prio=prio, loc_fn=loc_fn, loc_line=loc_line, loc_col=loc_col, )) logging.info( 'module %s training data conversion done. %d unique training samples.' % (module_name, len(td_list))) start_time = time.time() logging.info(u'bulk saving to db...') self.session.bulk_save_objects(td_list) self.session.commit() logging.info(u'bulk saving to db... done. Took %fs.' % (time.time() - start_time)) # put test data into our DB td_list = [] for name, lang, prep, rounds, loc_fn, loc_line, loc_col in tests: prep_json = prolog_to_json(prep) rounds_json = json.dumps(rounds) td_list.append( model.TestCase(lang=lang, module=module_name, name=name, prep=prep_json, rounds=rounds_json, loc_fn=loc_fn, loc_line=loc_line, loc_col=loc_col)) logging.info('module %s test data conversion done. %d tests.' % (module_name, len(td_list))) start_time = time.time() logging.info(u'bulk saving to db...') self.session.bulk_save_objects(td_list) self.session.commit() logging.info(u'bulk saving to db... done. Took %fs.' % (time.time() - start_time)) # put NER data into our DB # import pdb; pdb.set_trace() ner_list = [] for lang in ner: for cls in ner[lang]: for entity in ner[lang][cls]: ner_list.append( model.NERData(lang=lang, module=module_name, cls=cls, entity=entity, label=ner[lang][cls][entity])) logging.info('module %s NER data conversion done. %d rows.' % (module_name, len(ner_list))) start_time = time.time() logging.info(u'bulk saving to db...') self.session.bulk_save_objects(ner_list) self.session.commit() logging.info(u'bulk saving to db... done. Took %fs.' % (time.time() - start_time)) self.session.commit() def compile_module_multi(self, module_names): for module_name in module_names: if module_name == 'all': for mn2 in self.all_modules: self.load_module(mn2) self.compile_module(mn2) else: self.load_module(module_name) self.compile_module(module_name) self.session.commit() # _IGNORE_CONTEXT_KEYS = set([ 'user', 'lang', 'tokens', 'time', 'prev', 'resp' ]) def _compute_net_input(self, res, cur_context): solutions = self.rt.search_predicate('tokens', [cur_context, '_1'], env=res) tokens = solutions[0]['_1'].l solutions = self.rt.search_predicate('context', [cur_context, '_2', '_3'], env=res) d = {} for s in solutions: k = s['_2'] if not isinstance(k, Predicate): continue k = k.name v = s['_3'] if isinstance(v, Predicate): v = v.name elif isinstance(v, StringLiteral): v = v.s else: v = text_type(v) d[k] = v # import pdb; pdb.set_trace() inp = [] for t in reversed(tokens): inp.insert(0, t.s) for k in sorted(list(d)): inp.insert(0, [k, d[k]]) return inp def find_prev_context(self, user, env={}): pc = None ctxid = 0 # logging.debug ('find_prev_context: user=%s' % user) for s in self.rt.search_predicate('user', ['_1', Predicate(user)], env=env): cid = int(s['_1'].name[7:]) if not pc or cid > ctxid: pc = s['_1'] # logging.debug ('find_prev_context: s=%s, pc=%s' % (unicode(s), unicode(pc))) return pc def _setup_context(self, user, lang, inp, prev_context, prev_res): cur_context = Predicate(do_gensym(self.rt, 'context')) res = {} if ASSERT_OVERLAY_VAR_NAME in prev_res: res[ASSERT_OVERLAY_VAR_NAME] = prev_res[ ASSERT_OVERLAY_VAR_NAME].clone() res = do_assertz( {}, Clause(Predicate('user', [cur_context, Predicate(user)]), location=self.dummyloc), res=res) res = do_assertz( {}, Clause(Predicate('lang', [cur_context, Predicate(lang)]), location=self.dummyloc), res=res) token_literal = ListLiteral(list(map(lambda x: StringLiteral(x), inp))) res = do_assertz({}, Clause(Predicate('tokens', [cur_context, token_literal]), location=self.dummyloc), res=res) currentTime = datetime.datetime.utcnow().replace( tzinfo=pytz.UTC).isoformat() res = do_assertz( {}, Clause(Predicate( 'time', [cur_context, StringLiteral(currentTime)]), location=self.dummyloc), res=res) if prev_context: res = do_assertz({}, Clause(Predicate('prev', [cur_context, prev_context]), location=self.dummyloc), res=res) # copy over all previous context statements to the new one s1s = self.rt.search_predicate('context', [prev_context, '_1', '_2'], env=res) for s1 in s1s: res = do_assertz( {}, Clause(Predicate('context', [cur_context, s1['_1'], s1['_2']]), location=self.dummyloc), res=res) # copy over all previous mem statements to the new one s1s = self.rt.search_predicate('mem', [prev_context, '_1', '_2'], env=res) for s1 in s1s: res = do_assertz({}, Clause(Predicate( 'mem', [cur_context, s1['_1'], s1['_2']]), location=self.dummyloc), res=res) # import pdb; pdb.set_trace() res['C'] = cur_context return res, cur_context def _extract_response(self, cur_context, env): #import pdb; pdb.set_trace() res = [] s2s = self.rt.search_predicate('c_say', [cur_context, '_1'], env=env) for s2 in s2s: if not '_1' in s2: continue res.append(s2['_1'].s) actions = [] s2s = self.rt.search_predicate('c_action', [cur_context, '_1'], env=env) for s2 in s2s: if not '_1' in s2: continue actions.append(list(map(lambda x: text_type(x), s2['_1'].l))) score = 0.0 s2s = self.rt.search_predicate('c_score', [cur_context, '_1'], env=env) for s2 in s2s: if not '_1' in s2: continue score += s2['_1'].f return res, actions, score def _reconstruct_prolog_code(self, acode): todo = [('and', [])] idx = 0 while idx < len(acode): a = acode[idx] if a == 'or(': todo.append(('or', [])) elif a == 'and(': todo.append(('and', [])) elif a == ')': c = todo.pop() todo[len(todo) - 1][1].append(Predicate(c[0], c[1])) else: clause = self.aip_parser.parse_line_clause_body(a) todo[len(todo) - 1][1].append(clause.body) idx += 1 if len(todo) != 1: logging.warn('unbalanced acode detected.') return None c = todo.pop() return Predicate(c[0], c[1]) def test_module(self, module_name, run_trace=False, test_name=None): self.rt.set_trace(run_trace) m = self.modules[module_name] logging.info('running tests of module %s ...' % (module_name)) num_tests = 0 num_fails = 0 for tc in self.session.query( model.TestCase).filter(model.TestCase.module == module_name): if test_name: if tc.name != test_name: logging.info('skipping test %s' % tc.name) continue num_tests += 1 rounds = json.loads(tc.rounds) prep = json_to_prolog(tc.prep) round_num = 0 prev_context = None res = {} for t_in, t_out, test_actions in rounds: test_in = u' '.join(t_in) test_out = u' '.join(t_out) logging.info("nlp_test: %s round %d test_in : %s" % (tc.name, round_num, repr(test_in))) logging.info("nlp_test: %s round %d test_out : %s" % (tc.name, round_num, repr(test_out))) logging.info("nlp_test: %s round %d test_actions: %s" % (tc.name, round_num, repr(test_actions))) #if round_num>0: # import pdb; pdb.set_trace() res, cur_context = self._setup_context( user=TEST_USER, lang=tc.lang, inp=t_in, prev_context=prev_context, prev_res=res) # prep if prep: # import pdb; pdb.set_trace() # self.rt.set_trace(True) for p in prep: solutions = self.rt.search(Clause( None, p, location=self.dummyloc), env=res) if len(solutions) != 1: raise (PrologRuntimeError( 'Expected exactly one solution from preparation code for test "%s", got %d.' % (tc.name, len(solutions)))) res = solutions[0] # inp / resp inp = self._compute_net_input(res, cur_context) # look up code in DB acode = None matching_resp = False for tdr in self.session.query(model.TrainingData).filter( model.TrainingData.lang == tc.lang, model.TrainingData.inp == json.dumps(inp)): if acode: logging.warn( u'%s: more than one acode for test_in "%s" found in DB!' % (tc.name, test_in)) acode = json.loads(tdr.resp) pcode = self._reconstruct_prolog_code(acode) clause = Clause(None, pcode, location=self.dummyloc) solutions = self.rt.search(clause, env=res) # import pdb; pdb.set_trace() for solution in solutions: actual_out, actual_actions, score = self._extract_response( cur_context, solution) # logging.info("nlp_test: %s round %d %s" % (clause.location, round_num, repr(abuf)) ) if len(test_out) > 0: if len(actual_out) > 0: actual_out = u' '.join( tokenize(u' '.join(actual_out), tc.lang)) logging.info( "nlp_test: %s round %d actual_out : %s (score: %f)" % (tc.name, round_num, actual_out, score)) if actual_out != test_out: logging.info( "nlp_test: %s round %d UTTERANCE MISMATCH." % (tc.name, round_num)) continue # no match logging.info( "nlp_test: %s round %d UTTERANCE MATCHED!" % (tc.name, round_num)) # check actions if len(test_actions) > 0: logging.info( "nlp_test: %s round %d actual acts : %s" % (tc.name, round_num, repr(actual_actions))) # print repr(test_actions) actions_matched = True act = None for action in test_actions: for act in actual_actions: # print " check action match: %s vs %s" % (repr(action), repr(act)) if action == act: break if action != act: actions_matched = False break if not actions_matched: logging.info( "nlp_test: %s round %d ACTIONS MISMATCH." % (tc.name, round_num)) continue logging.info( "nlp_test: %s round %d ACTIONS MATCHED!" % (tc.name, round_num)) matching_resp = True res = solution break if matching_resp: break if acode is None: logging.error('failed to find db entry for %s' % json.dumps(inp)) logging.error( u'Error: %s: no training data for test_in "%s" found in DB!' % (tc.name, test_in)) num_fails += 1 break if not matching_resp: logging.error( u'nlp_test: %s round %d no matching response found.' % (tc.name, round_num)) num_fails += 1 break prev_context = cur_context round_num += 1 self.rt.set_trace(False) return num_tests, num_fails def run_tests_multi(self, module_names, run_trace=False, test_name=None): num_tests = 0 num_fails = 0 for module_name in module_names: if module_name == 'all': for mn2 in self.all_modules: self.load_module(mn2) self.init_module(mn2, run_trace=run_trace) n, f = self.test_module(mn2, run_trace=run_trace, test_name=test_name) num_tests += n num_fails += f else: self.load_module(module_name) self.init_module(module_name, run_trace=run_trace) n, f = self.test_module(module_name, run_trace=run_trace, test_name=test_name) num_tests += n num_fails += f return num_tests, num_fails def _process_input_nnet(self, inp, res): solutions = [] logging.debug('_process_input_nnet: %s' % repr(inp)) try: # ok, exact matching has not yielded any results -> use neural network to # generate response(s) x = self.nlp_model.compute_x(inp) # logging.debug("x: %s -> %s" % (utterance, x)) source, source_len, dest, dest_len = self.nlp_model._prepare_batch( [[x, []]], offset=0) # predicted_ids: GreedyDecoder; [batch_size, max_time_step, 1] # BeamSearchDecoder; [batch_size, max_time_step, beam_width] predicted_ids = self.tf_model.predict( self.tf_session, encoder_inputs=source, encoder_inputs_length=source_len) # for seq_batch in predicted_ids: # for k in range(5): # logging.debug('--------- k: %d ----------' % k) # seq = seq_batch[:,k] # for p in seq: # if p == -1: # break # decoded = self.inv_output_dict[p] # logging.debug (u'%s: %s' %(p, decoded)) # extract best codes only acodes = [[]] for p in predicted_ids[0][:, 0]: if p == -1: break decoded = self.inv_output_dict[p] if decoded == u'_EOS': break if decoded == u'__OR__': acodes.append([]) acodes[len(acodes) - 1].append(decoded) # FIXME: for now, we try the first solution only acode = acodes[0] pcode = self._reconstruct_prolog_code(acode) logging.debug('_process_input_nnet: %s' % pcode) clause = Clause(None, pcode, location=self.dummyloc) solutions = self.rt.search(clause, env=res) except: # probably ok (prolog code generated by neural network might not always work) logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) return solutions def process_input(self, utterance, utt_lang, user_uri, run_trace=False, do_eliza=True, prev_ctx=None): """ process user input, return score, responses, actions, solutions, context """ prev_context = prev_ctx res = {} tokens = tokenize(utterance, utt_lang) res, cur_context = self._setup_context(user=user_uri, lang=utt_lang, inp=tokens, prev_context=prev_context, prev_res=res) inp = self._compute_net_input(res, cur_context) logging.debug('process_input: %s' % repr(inp)) # # do we have an exact match in our training data for this input? # solutions = [] self.rt.set_trace(run_trace) for tdr in self.session.query(model.TrainingData).filter( model.TrainingData.lang == utt_lang, model.TrainingData.inp == json.dumps(inp)): acode = json.loads(tdr.resp) pcode = self._reconstruct_prolog_code(acode) clause = Clause(None, pcode, location=self.dummyloc) sols = self.rt.search(clause, env=res) if sols: solutions.extend(sols) if not solutions: solutions = self._process_input_nnet(inp, res) # # try dropping the context if we haven't managed to produce a result yet # if not solutions: res, cur_context = self._setup_context(user=user_uri, lang=utt_lang, inp=tokens, prev_context=None, prev_res={}) inp = self._compute_net_input(res, cur_context) solutions = self._process_input_nnet(inp, res) if not solutions and do_eliza: logging.info('producing ELIZA-style response for input %s' % utterance) clause = self.aip_parser.parse_line_clause_body( 'do_eliza(C, %s)' % utt_lang) solutions = self.rt.search(clause, env=res) self.rt.set_trace(False) # # extract highest-scoring responses only: # best_score = 0 best_resps = [] best_actions = [] best_solutions = [] for solution in solutions: actual_resp, actual_actions, score = self._extract_response( cur_context, solution) if score > best_score: best_score = score best_resps = [] best_actions = [] best_solutions = [] if score < best_score: continue best_resps.append(actual_resp) best_actions.append(actual_actions) best_solutions.append(solution) return best_score, best_resps, best_actions, best_solutions, cur_context def run_cronjobs(self, module_name, force=False, run_trace=False): m = self.modules[module_name] if not hasattr(m, 'CRONJOBS'): return self.rt.set_trace(run_trace) for name, interval, f in getattr(m, 'CRONJOBS'): cronjob = self.session.query(model.Cronjob).filter( model.Cronjob.module == module_name, model.Cronjob.name == name).first() t = time.time() next_run = cronjob.last_run + interval if force or t > next_run: logging.debug('running cronjob %s' % name) f(self) cronjob.last_run = t def run_cronjobs_multi(self, module_names, force, run_trace=False): for module_name in module_names: if module_name == 'all': for mn2 in self.all_modules: self.load_module(mn2) self.init_module(mn2) self.run_cronjobs(mn2, force=force, run_trace=run_trace) else: self.load_module(module_name) self.init_module(module_name) self.run_cronjobs(module_name, force=force, run_trace=run_trace) self.session.commit() def train(self, ini_fn, num_steps, incremental): self.setup_tf_model('train', False, ini_fn) self.nlp_model.train(num_steps, incremental) def dump_utterances(self, num_utterances, dictfn, lang, module): dic = None if dictfn: dic = set() with codecs.open(dictfn, 'r', 'utf8') as dictf: for line in dictf: parts = line.strip().split(';') if len(parts) != 2: continue dic.add(parts[0]) all_utterances = [] req = self.session.query( model.TrainingData).filter(model.TrainingData.lang == lang) if module and module != 'all': req = req.filter(model.TrainingData.module == module) for dr in req: if not dic: all_utterances.append(dr.utterance) else: # is at least one word not covered by our dictionary? unk = False for t in tokenize(dr.utterance): if not t in dic: # print u"unknown word: %s in %s" % (t, dr.utterance) unk = True dic.add(t) break if not unk: continue all_utterances.append(dr.utterance) utts = set() if num_utterances > 0: while (len(utts) < num_utterances): i = random.randrange(0, len(all_utterances)) utts.add(all_utterances[i]) else: for utt in all_utterances: utts.add(utt) for utt in utts: print(utt) def setup_align_utterances(self, lang): if self.w2v_model and self.w2v_lang == lang: return logging.debug('loading all utterances from db...') self.w2v_all_utterances = [] req = self.session.query( model.TrainingData).filter(model.TrainingData.lang == lang) for dr in req: self.w2v_all_utterances.append( (dr.utterance, dr.module, dr.loc_fn, dr.loc_line)) if not self.w2v_model: from gensim.models import word2vec model_fn = self.config.get('semantics', 'word2vec_model_%s' % lang) logging.debug('loading word2vec model %s ...' % model_fn) logging.getLogger('gensim.models.word2vec').setLevel(logging.WARNING) self.w2v_model = word2vec.Word2Vec.load_word2vec_format(model_fn, binary=True) self.w2v_lang = lang #list containing names of words in the vocabulary self.w2v_index2word_set = set(self.w2v_model.index2word) logging.debug('loading word2vec model %s ... done' % model_fn) def align_utterances(self, lang, utterances): self.setup_align_utterances(lang) res = {} for utt1 in utterances: try: utt1t = tokenize(utt1, lang=lang) av1 = avg_feature_vector( utt1t, model=self.w2v_model, num_features=300, index2word_set=self.w2v_index2word_set) sims = {} # location -> score utts = {} # location -> utterance for utt2, module, loc_fn, loc_line in self.w2v_all_utterances: try: utt2t = tokenize(utt2, lang=lang) av2 = avg_feature_vector( utt2t, model=self.w2v_model, num_features=300, index2word_set=self.w2v_index2word_set) sim = 1 - cosine(av1, av2) location = '%s:%s:%d' % (module, loc_fn, loc_line) sims[location] = sim utts[location] = utt2 # logging.debug('%10.8f %s' % (sim, location)) except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) logging.info('sims for %s' % repr(utt1)) cnt = 0 res[utt1] = [] for sim, location in sorted( ((v, k) for k, v in sims.iteritems()), reverse=True): logging.info('%10.8f %s' % (sim, location)) logging.info(' %s' % (utts[location])) res[utt1].append((sim, location, utts[location])) cnt += 1 if cnt > 5: break except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) return res
class AIKernal(object): def __init__(self, db_url, xsb_root, toplevel): # # database connection # self.engine = create_engine(db_url, echo=False) self.Session = sessionmaker(bind=self.engine) self.session = self.Session() model.Base.metadata.create_all(self.engine) # # TensorFlow (deferred, as tf can take quite a bit of time to set up) # self.tf_session = None self.nlp_model = None # # module management, setup # self.modules = {} self.consulted_modules = set() self.toplevel = toplevel self.all_modules = [] sys.path.append('modules') self.load_module(toplevel) # # Prolog engine, data engine # xsb_hl_init([xsb_root]) self.dte = DataEngine(self.session) xsb_hl_command( 'import default_sys_error_handler/1 from error_handler.') xsb_hl_command( 'assertz((default_user_error_handler(Ball):-default_sys_error_handler(Ball))).' ) # # restore memory # q = u'' for m in self.session.query(model.Mem): v = json_to_xsb(m.v) if q: q += ', ' q += u"assertz(memory('%s', '%s', %s, %f))" % (m.realm, m.k, unicode(v), m.score) if not q: q = u'assertz(memory(self, self, self, 1.0))' q += u'.' xsb_hl_command(q) # import pdb; pdb.set_trace() # # alignment / word2vec (on-demand model loading) # self.w2v_model = None self.w2v_lang = None self.w2v_all_utterances = [] # FIXME: this will work only on the first call def setup_tf_model(self, mode, load_model, ini_fn, global_step=0): if not self.tf_session: import tensorflow as tf # setup config to use BFC allocator config = tf.ConfigProto() # config.gpu_options.allocator_type = 'BFC' self.tf_session = tf.Session(config=config) if not self.nlp_model: from nlp_model import NLPModel self.nlp_model = NLPModel(self.session, ini_fn, global_step=global_step) if load_model: self.nlp_model.load_dicts() # we need the inverse dict to reconstruct the output from tensor self.inv_output_dict = { v: k for k, v in viewitems(self.nlp_model.output_dict) } self.tf_model = self.nlp_model.create_tf_model(self.tf_session, mode=mode) self.tf_model.batch_size = 1 self.tf_model.restore(self.tf_session, self.nlp_model.model_fn) def clean(self, module_names): for module_name in module_names: self.dte.clean(module_name) self.session.commit() def load_module(self, module_name): if module_name in self.modules: return self.modules[module_name] logging.debug("loading module '%s'" % module_name) # fp, pathname, description = imp.find_module(module_name, ['modules']) fp, pathname, description = imp.find_module(module_name) # print fp, pathname, description m = None try: m = imp.load_module(module_name, fp, pathname, description) self.modules[module_name] = m for m2 in getattr(m, 'DEPENDS'): self.load_module(m2) except: logging.error('failed to load module "%s"' % module_name) logging.error(traceback.format_exc()) sys.exit(1) finally: # Since we may exit via an exception, close fp explicitly. if fp: fp.close() if not module_name in self.all_modules: self.all_modules.append(module_name) return m def consult_module(self, module_name): if module_name in self.consulted_modules: return logging.debug("consulting module '%s'" % module_name) m = self.load_module(module_name) self.consulted_modules.add(module_name) try: # print m # print getattr(m, '__all__', None) # for name in dir(m): # print name for m2 in getattr(m, 'DEPENDS'): self.consult_module(m2) if hasattr(m, 'PL_SOURCES'): for inputfn in m.PL_SOURCES: pl_path = "modules/%s/%s" % (module_name, inputfn) xsb_hl_command("consult('%s')." % pl_path) except: logging.error('failed to load module "%s"' % module_name) logging.error(traceback.format_exc()) sys.exit(1) return m def compile_module(self, module_name): m = self.load_module(module_name) # tell prolog engine to consult all prolog files plus their dependencies self.consult_module(module_name) # prepare data engine for module compilation self.dte.prepare_compilation(module_name) if hasattr(m, 'get_data'): logging.info('module %s data extraction...' % module_name) get_data = getattr(m, 'get_data') get_data(self) self.dte.commit() cnt_dt, cnt_ts = self.dte.get_stats() logging.info( 'module %s data extraction done. %d training samples, %d tests' % (module_name, cnt_dt, cnt_ts)) def compile_module_multi(self, module_names): for module_name in module_names: if module_name == 'all': for mn2 in self.all_modules: self.compile_module(mn2) else: self.compile_module(module_name) def test_module(self, module_name, run_trace=False, test_name=None): if run_trace: xsb_hl_command("trace.") else: xsb_hl_command("notrace.") m = self.modules[module_name] logging.info('running tests of module %s ...' % (module_name)) num_tests = 0 num_fails = 0 for tc in self.dte.lookup_tests(module_name): t_name, lang, prep_code, prep_fn, rounds, src_fn, self.src_line = tc if test_name: if t_name != test_name: logging.info('skipping test %s' % t_name) continue ctx = AIContext(TEST_USER, self.session, lang, TEST_REALM, self, test_mode=True) round_num = 0 num_tests += 1 self.mem_clear(TEST_REALM) self.mem_clear(TEST_USER) # prep if prep_code: pcode = '%s\n%s(ctx)\n' % (prep_code, prep_fn) try: exec(pcode, globals(), locals()) except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) for test_inp, test_out, test_action, test_action_arg in rounds: logging.info("test_module: %s round %d test_inp : %s" % (t_name, round_num, repr(test_inp))) logging.info("test_module: %s round %d test_out : %s" % (t_name, round_num, repr(test_out))) # look up code in data engine matching_resp = False acode = None ctx.set_inp(test_inp) self.mem_set(ctx.realm, 'action', None) for lang, d, md5s, args, src_fn, src_line in self.dte.lookup_data_train( test_inp, lang): afn, acode = self.dte.lookup_code(md5s) ecode = '%s\n%s(ctx' % (acode, afn) if args: for arg in args: ecode += ',%s' % repr(arg) ecode += ')\n' # import pdb; pdb.set_trace() try: exec(ecode, globals(), locals()) except: logging.error( 'test_module: %s round %d EXCEPTION CAUGHT %s' % (t_name, round_num, traceback.format_exc())) logging.error(ecode) if acode is None: logging.error( u'Error: %s: no training data for test_in "%s" found in DB!' % (t_name, test_inp)) num_fails += 1 break resps = ctx.get_resps() for i, resp in enumerate(resps): actual_out, score, actual_action, actual_action_arg = resp # logging.info("test_module: %s round %d %s" % (clause.location, round_num, repr(abuf)) ) if len(test_out) > 0: if len(actual_out) > 0: actual_out = u' '.join(tokenize(actual_out, lang)) logging.info( "test_module: %s round %d actual_out : %s (score: %f)" % (t_name, round_num, actual_out, score)) if actual_out != test_out: logging.info( "test_module: %s round %d UTTERANCE MISMATCH." % (t_name, round_num)) continue # no match logging.info( "test_module: %s round %d UTTERANCE MATCHED!" % (t_name, round_num)) matching_resp = True ctx.commit_resp(i) # check action if test_action: afn, acode = self.dte.lookup_code(test_action) ecode = '%s\n%s(ctx' % (acode, afn) if test_action_arg: ecode += ',%s' % repr(test_action_arg) ecode += ')\n' exec(ecode, globals(), locals()) break if not matching_resp: logging.error( u'test_module: %s round %d no matching response found.' % (t_name, round_num)) num_fails += 1 break round_num += 1 return num_tests, num_fails def run_tests_multi(self, module_names, run_trace=False, test_name=None): num_tests = 0 num_fails = 0 for module_name in module_names: if module_name == 'all': for mn2 in self.all_modules: self.consult_module(mn2) n, f = self.test_module(mn2, run_trace=run_trace, test_name=test_name) num_tests += n num_fails += f else: self.consult_module(module_name) n, f = self.test_module(module_name, run_trace=run_trace, test_name=test_name) num_tests += n num_fails += f return num_tests, num_fails def process_input(self, ctx, inp_raw, inp_lang, user_uri, run_trace=False, do_eliza=True): """ process user input, return score, responses, actions, solutions, context """ if run_trace: xsb_hl_command("trace.") else: xsb_hl_command("notrace.") tokens_raw = tokenize(inp_raw, inp_lang) tokens = [] for t in tokens_raw: if t == u'nspc': continue tokens.append(t) inp = u" ".join(tokens) ctx.set_inp(inp) self.mem_set(ctx.realm, 'action', None) logging.debug( '===============================================================================' ) logging.debug('process_input: %s' % repr(inp)) # # do we have an exact match in our training data for this input? # found_resp = False for lang, d, md5s, args, src_fn, src_line in self.dte.lookup_data_train( inp, inp_lang): afn, acode = self.dte.lookup_code(md5s) ecode = '%s\n%s(ctx' % (acode, afn) if args: for arg in args: ecode += ',%s' % repr(arg) ecode += ')\n' logging.debug('exact training data match found: %s:%s' % (src_fn, src_line)) logging.debug(ecode) # import pdb; pdb.set_trace() try: exec(ecode, globals(), locals()) found_resp = True except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) logging.error(ecode) if not found_resp: logging.debug('no exact training data match for this input found.') # # ask neural net if we did not find an answer # resps = ctx.get_resps() if not resps and self.nlp_model: from nlp_model import OR_SYMBOL from seq2seq_model import _EOS logging.debug('trying neural net on: %s' % repr(inp)) try: # ok, exact matching has not yielded any results -> use neural network to # generate response(s) x = self.nlp_model.compute_x(inp) # logging.debug("x: %s -> %s" % (utterance, x)) source, source_len, dest, dest_len = self.nlp_model._prepare_batch( [[x, []]], offset=0) # predicted_ids: GreedyDecoder; [batch_size, max_time_step, 1] # BeamSearchDecoder; [batch_size, max_time_step, beam_width] predicted_ids = self.tf_model.predict( self.tf_session, encoder_inputs=source, encoder_inputs_length=source_len) # for seq_batch in predicted_ids: # for k in range(5): # logging.debug('--------- k: %d ----------' % k) # seq = seq_batch[:,k] # for p in seq: # if p == -1: # break # decoded = self.inv_output_dict[p] # logging.debug (u'%s: %s' %(p, decoded)) # extract best codes, run them all to see which ones yield the highest scoring responses cmd = [] for p in predicted_ids[0][:, 0]: if p in self.inv_output_dict: decoded = self.inv_output_dict[p] else: decoded = p if decoded == _EOS or decoded == -1 or decoded == OR_SYMBOL: try: logging.debug('trying cmd: %s' % repr(cmd)) afn, acode = self.dte.lookup_code(cmd[0]) ecode = '%s\n%s(ctx' % (acode, afn) if len(cmd) > 1: for arg in cmd[1:]: ecode += ',%s' % repr(json.loads(arg)) ecode += ')\n' logging.debug(ecode) exec(ecode, globals(), locals()) except: logging.debug('EXCEPTION CAUGHT %s' % traceback.format_exc()) cmd = [] if decoded == _EOS or decoded == -1: break else: cmd.append(decoded) except: # probably ok (prolog code generated by neural network might not always work) logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) # if not solutions: # res, cur_context = self._setup_context ( user = user_uri, # lang = inp_lang, # inp = tokens, # prev_context = None, # prev_res = {}) # inp = self._compute_net_input (res, cur_context) # solutions = self._process_input_nnet(inp, res) xsb_hl_command("notrace.") # # extract highest-scoring responses # resps = ctx.get_resps() if not resps and do_eliza: logging.debug('producing ELIZA-style response for input %s' % inp) from psychology import psychology psychology.do_eliza(ctx) resps = ctx.get_resps() # # pick random response # if len(resps) > 0: i = random.randrange(0, len(resps)) out, score, action, action_arg = resps[i] ctx.commit_resp(i) logging.debug(u'picked resp #%d (score: %f): %s' % (i, score, out)) logging.debug(u'MEM: %s' % ctx.realm) memd = self.mem_dump(ctx.realm) for k, v, score in memd: logging.debug(u'MEM: %-20s: %s (%f)' % (k, v, score)) logging.debug(u'MEM: %s' % user_uri) memd = self.mem_dump(user_uri) for k, v, score in memd: logging.debug(u'MEM: %-20s: %s (%f)' % (k, v, score)) else: out = u'' score = 0.0 logging.debug(u'No response found.') action = self.mem_get(ctx.realm, 'action') return out, score, action def train(self, ini_fn, num_steps, incremental): self.setup_tf_model('train', False, ini_fn) self.nlp_model.train(num_steps, incremental) def dump_utterances(self, num_utterances, dictfn, lang, module): dic = None if dictfn: dic = set() with codecs.open(dictfn, 'r', 'utf8') as dictf: for line in dictf: parts = line.strip().split(';') if len(parts) != 2: continue dic.add(parts[0]) req = self.dte.session.query( model.TrainingData).filter(model.TrainingData.lang == lang) if module and module != 'all': req = req.filter(model.TrainingData.module == module) req_utts = [] for dr in req: req_utts.append(dr.inp) if not dic: all_utterances = sorted(req_utts) else: all_utterances = [] random.shuffle(req_utts) for utt in req_utts: # is at least one word not covered by our dictionary? unk = False for t in tokenize(utt): if not t in dic: # print u"unknown word: %s in %s" % (t, utt) unk = True break if not unk: continue for t in tokenize(utt): dic.add(t) all_utterances.append(utt) utts = set() if num_utterances > 0: while (len(utts) < num_utterances): i = random.randrange(0, len(all_utterances)) utts.add(all_utterances[i]) else: for utt in all_utterances: utts.add(utt) for utt in sorted(list(utts)): print(utt) def setup_align_utterances(self, lang): if self.w2v_model and self.w2v_lang == lang: return logging.debug('loading all utterances from db...') self.w2v_all_utterances = [] req = self.session.query( model.TrainingData).filter(model.TrainingData.lang == lang) for dr in req: self.w2v_all_utterances.append( (dr.utterance, dr.module, dr.loc_fn, dr.loc_line)) if not self.w2v_model: from gensim.models import word2vec model_fn = self.config.get('semantics', 'word2vec_model_%s' % lang) logging.debug('loading word2vec model %s ...' % model_fn) logging.getLogger('gensim.models.word2vec').setLevel(logging.WARNING) self.w2v_model = word2vec.Word2Vec.load_word2vec_format(model_fn, binary=True) self.w2v_lang = lang #list containing names of words in the vocabulary self.w2v_index2word_set = set(self.w2v_model.index2word) logging.debug('loading word2vec model %s ... done' % model_fn) def align_utterances(self, lang, utterances): self.setup_align_utterances(lang) res = {} for utt1 in utterances: try: utt1t = tokenize(utt1, lang=lang) av1 = avg_feature_vector( utt1t, model=self.w2v_model, num_features=300, index2word_set=self.w2v_index2word_set) sims = {} # location -> score utts = {} # location -> utterance for utt2, module, loc_fn, loc_line in self.w2v_all_utterances: try: utt2t = tokenize(utt2, lang=lang) av2 = avg_feature_vector( utt2t, model=self.w2v_model, num_features=300, index2word_set=self.w2v_index2word_set) sim = 1 - cosine(av1, av2) location = '%s:%s:%d' % (module, loc_fn, loc_line) sims[location] = sim utts[location] = utt2 # logging.debug('%10.8f %s' % (sim, location)) except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) logging.info('sims for %s' % repr(utt1)) cnt = 0 res[utt1] = [] for sim, location in sorted( ((v, k) for k, v in sims.iteritems()), reverse=True): logging.info('%10.8f %s' % (sim, location)) logging.info(' %s' % (utts[location])) res[utt1].append((sim, location, utts[location])) cnt += 1 if cnt > 5: break except: logging.error('EXCEPTION CAUGHT %s' % traceback.format_exc()) return res def stats(self): stats = {} for module_name in self.all_modules: stats[module_name] = {} for lang in LANGUAGES: cnt = self.session.query(model.TrainingData).filter( model.TrainingData.module == module_name, model.TrainingData.lang == lang).count() stats[module_name][lang] = cnt return stats def mem_clear(self, realm): if not isinstance(realm, basestring): raise Exception("mem_set: realm must be string-typed.") q = u"retractall(memory('%s', _, _, _))." % realm # logging.debug (q) self.prolog_query(q) def mem_dump(self, realm): if not isinstance(realm, basestring): raise Exception("mem_set: realm must be string-typed.") entries = [] q = u"memory('%s', K, V, S)." % realm # logging.debug (q) res = self.prolog_query(q) if res: for r in res: k = r[0] v = r[1] score = r[2] entries.append((k, v, score)) # import pdb; pdb.set_trace() return entries def mem_set(self, realm, k, v): if not isinstance(realm, basestring) or not isinstance(k, basestring): raise Exception("mem_set: realm and key must be string-typed.") q = u"retractall(memory('%s', '%s', _, _))" % (realm, k) if v: q += u", assertz(memory('%s', '%s', %s, 1.0))." % (realm, k, unicode(v)) else: q += '.' # logging.debug (q) self.prolog_query(q) def mem_get(self, realm, k): if not isinstance(realm, basestring) or not isinstance(k, basestring): raise Exception("mem_set: realm and key must be string-typed.") q = u"memory('%s', '%s', V, S)." % (realm, k) # logging.debug (q) res = self.prolog_query(q) if not res: return None score = 0.0 v = None for r in res: if not v or r[1] > score: score = r[1] v = r[0] return v def mem_get_multi(self, realm, k): if not isinstance(realm, basestring) or not isinstance(k, basestring): raise Exception("mem_set: realm and key must be string-typed.") entries = [] q = u"memory('%s', '%s', V, S)." % (realm, k) # logging.debug (q) res = self.prolog_query(q) if res: for r in res: score = r[1] v = r[0] entries.append((v, score)) return entries def mem_push(self, realm, k, v): if not isinstance(realm, basestring) or not isinstance(k, basestring): raise Exception("mem_set: realm and key must be string-typed.") entries = [(1.0, v)] # re-score existing entries q = u"memory('%s', '%s', V, S)." % (realm, k) # logging.debug (q) res = self.prolog_query(q) if res: for r in res: score = r[1] v = r[0] if score < 0.125: continue entries.append((score / 2, v)) # put all entries into the KB q = u"retractall(memory('%s', '%s', _, _))" % (realm, k) for score, v in entries: if v: q += u", assertz(memory('%s', '%s', %s, %f))" % ( realm, k, unicode(v), score) q += u'.' # logging.debug (q) self.prolog_query(q) def prolog_query(self, query): logging.debug('prolog_query: %s' % query) return xsb_hl_query(query) def prolog_check(self, query): logging.debug('prolog_check: %s' % query) res = xsb_hl_query(query) return len(res) > 0 def prolog_query_one(self, query, idx=0): logging.debug('prolog_query_one: %s' % query) solutions = xsb_hl_query(query) if not solutions: return None return solutions[0][idx] def prolog_persist(self): """ persist all currently asserted dynamic memory predicates from the prolog KB """ # q = 'predicate_property(Head, (dynamic)), clause(Head,Bod).' # for r in self.prolog_query(q): # logging.info(repr(r)) # # print "%s" % repr(r) self.session.query(model.Mem).delete() q = u'memory(REALM, K, V, S).' for r in self.prolog_query(q): # logging.info(repr(r)) realm = r[0].name k = r[1].name v = xsb_to_json(r[2]) score = r[3] m = model.Mem(realm=realm, k=k, v=v, score=score) self.session.add(m) # import pdb; pdb.set_trace() self.session.commit()