def __init__(self): self.history_savedir = None self.detail_savedir = None self.logger = None self.user = self.create_user() self.rule_policy = RulePolicy() self.dst = DialogStateTracker(UserPersonal, FLAGS.print, self.logger) self.data_manager = DataManager(os.path.join(BASE_DIR, 'data/tmp')) self.nlu_manager = NLUManager(NLU_save_path_dict) # self.nlg_template = NLG_template self.turn_num = 1 self.dialog_history = []
class OnlineStoreApp(App): dataManager = DataManager() ordersList = ListProperty([]) def app_func(self): '''Wrapper functions for the async processes. ''' def run_wrapper(): # Run the Kivy UI self.async_run() exit(0) return asyncio.gather(run_wrapper()) def __init__(self, **kwargs): super().__init__(**kwargs) def build(self): return OrderScreen()
def setUp(self): removeLeftOverDbFiles() DataManagerUnitTest.dm = DataManager( configObj=DataManagerUnitTest.testConfig)
if 'chatting' in SysAct.keys(): strategy = random.choice([0,1,2]) if strategy == 0: nl += '您好,我是中国移动业务咨询机器人,可以帮助您进行套餐查找、业务咨询、个人信息查询等,请问您需要什么帮助吗?' elif strategy == 1: nl += '抱歉,您说的这些我不太理解,但我可以在套餐查找、业务咨询、个人信息查询等方面为您提供帮助的~' else: nl += '请不要再调戏我了,问一些套餐查找、业务咨询、个人信息查询方面的问题吧' # if 'repeat' in SysAct.keys(): # nl += "抱歉,能再重述一遍么?" return nl if __name__ == '__main__': data_manager = DataManager('../data/tmp') offer_entity = data_manager.SearchingByConstraints('套餐', {"功能费": [700, 900]})[0] compared_entities = data_manager.SearchingByConstraints('套餐', {"功能费": [700, 900]}) domain = '套餐' SysAct = { 'ex0': { 'offer': offer_entity, 'inform':['产品介绍', '套餐内容_国内主叫', '套餐内容_国内流量', '套餐内容_国内短信', '结转规则', '是否包含港澳台地区']}, 'ex1': #self.UsrAct == "告知": {'offer': offer_entity, 'inform': ["产品介绍"], 'reqmore': None, # None 是因为 reqmore 没有参数 'domain':domain}, 'ex2': {'request': ["套餐内容_国内流量", "套餐内容_国内主叫"], 'domain': domain},
def train_requestable(data_tmp_path): """ 用于训练模型,先训练完存好了才能用 训练用 early stopping :param data_tmp_path: data tmp 文件夹位置 """ print('载入数据管理器...') data_manager = DataManager(data_tmp_path) print('载入训练数据...') informable_slot_datasets, requestable_slot_datasets = generate_dataset( data_manager.DialogData) print('载入 requestable slot detector...') init_learning_rate = 0.005 graph = tf.Graph() with graph.as_default(): requestable_slots_models = {} for k, v in All_requestable_slots_order.items(): requestable_slots_models[k] = RequestableSlotDector( str(v), learning_rate=init_learning_rate) with tf.Session(graph=graph, config=tf.ConfigProto(allow_soft_placement=True)) as sess: sess.run(tf.group(tf.global_variables_initializer())) saver = tf.train.Saver() # saver.restore(sess, "./ckpt/requestable/model.ckpt") # 训练 requestable slots requestable_slots_F1s = {} for slot, model in requestable_slots_models.items(): average_loss = 0 learning_rate = init_learning_rate best_F1 = 0 tolerance = 30 tolerance_count = 0 display_step = 10 for step in range(5000): step += 1 batch_data, batch_output = requestable_slot_datasets[ slot].next_batch() char_emb_matrix, word_emb_matrix, seqlen = data_manager.sent2num( batch_data) _, training_loss = sess.run( [model.train_op, model.final_loss], feed_dict={ model.char_emb_matrix: char_emb_matrix, model.word_emb_matrix: word_emb_matrix, model.output: batch_output }) average_loss += training_loss / display_step if step % display_step == 0: batch_data, batch_output = requestable_slot_datasets[ slot].get_testset() char_emb_matrix, word_emb_matrix, seqlen = data_manager.sent2num( batch_data) pred, accu = sess.run( [model.predict, model.accuracy], feed_dict={ model.char_emb_matrix: char_emb_matrix, model.word_emb_matrix: word_emb_matrix, model.output: batch_output }) F1 = get_F1score(batch_output, pred.tolist()) if best_F1 < F1: best_F1 = F1 tolerance_count = 0 if not os.path.exists("./ckpt/requestable/"): os.makedirs("./ckpt/requestable/") saver.save(sess, "./ckpt/requestable/model.ckpt") if tolerance_count == tolerance: break print("%s, step % 4d, loss %0.4f, F1 %0.4f, accu %0.4f" % (slot, step, average_loss, F1, accu)) average_loss = 0 tolerance_count += 1 learning_rate = max(learning_rate * 0.98, 0.001) sess.run(model.update_lr, feed_dict={model.new_lr: learning_rate}) print("requestable slot: %s, best F1 %0.4f" % (slot, best_F1)) requestable_slots_F1s[slot] = best_F1 print(requestable_slots_F1s) print( sum(requestable_slots_F1s.values()) / len(requestable_slots_F1s.values()))
def train_informable(data_tmp_path): """ 用于训练模型,先训练完存好了才能用 训练用 early stopping :param data_tmp_path: data tmp 文件夹位置 """ print('载入数据管理器...') data_manager = DataManager(data_tmp_path) print('载入训练数据...') informable_slot_datasets, requestable_slot_datasets = generate_dataset( data_manager.DialogData) print('载入 informable slot detector ...') init_learning_rate = 0.005 informable_batch_ratios = { # 不同slot 的minibatch ratio "通话时长": [2, 8, 8, 8], "流量": [4, 8, 8, 8], "功能费": [4, 8, 8, 8] } graph = tf.Graph() with graph.as_default(): informable_slots_models = { "功能费": InformableSlotDector('cost', learning_rate=init_learning_rate), "流量": InformableSlotDector('data', learning_rate=init_learning_rate), "通话时长": InformableSlotDector('time', learning_rate=init_learning_rate), } with tf.Session(graph=graph, config=tf.ConfigProto(allow_soft_placement=True)) as sess: sess.run(tf.group(tf.global_variables_initializer())) saver = tf.train.Saver() # saver.restore(sess, "./ckpt/informable/model.ckpt") # 训练 informable slots informable_slots_accus = [] for slot, model in informable_slots_models.items(): learning_rate = init_learning_rate average_loss = 0 best_accu = 0 tolerance = 20 tolerance_count = 0 display_step = 10 for step in range(5000): step += 1 batch_data, batch_output = informable_slot_datasets[ slot].next_batch(informable_batch_ratios[slot]) char_emb_matrix, word_emb_matrix, seqlen = data_manager.sent2num( batch_data) _, training_loss = sess.run( [model.train_op, model.final_loss], feed_dict={ model.char_emb_matrix: char_emb_matrix, model.word_emb_matrix: word_emb_matrix, model.output: batch_output }) average_loss += training_loss / display_step if step % display_step == 0: batch_data, batch_output = informable_slot_datasets[ slot].get_testset() char_emb_matrix, word_emb_matrix, seqlen = data_manager.sent2num( batch_data) pred, accu = sess.run( [model.predict, model.accuracy], feed_dict={ model.char_emb_matrix: char_emb_matrix, model.word_emb_matrix: word_emb_matrix, model.output: batch_output }) if best_accu < accu: best_accu = accu tolerance_count = 0 if not os.path.exists("./ckpt/informable/"): os.makedirs("./ckpt/informable/") saver.save(sess, "./ckpt/informable/model.ckpt") if tolerance_count == tolerance: break print("%s, step % 4d, loss %0.4f, accu %0.4f" % (slot, step, average_loss, accu)) average_loss = 0 tolerance_count += 1 learning_rate = max(learning_rate * 0.95, 0.0001) sess.run(model.update_lr, feed_dict={model.new_lr: learning_rate}) print("informable slot: %s, best accu %0.4f" % (slot, best_accu)) informable_slots_accus.append(best_accu)
from datetime import datetime from data.DataManager import DataManager from structures.NeuralNetwork import NeuralNetwork from structures.functions import sigmoid, d_sigmoid, same, d_same from structures.layers.DenseLayer import DenseLayer def classify(arr): a, b = max((x, y) for y, x in enumerate(arr)) return b if __name__ == '__main__': dm = DataManager('mnist.pkl.gz') nn = NeuralNetwork() nn.add_layer(DenseLayer(784, 400, sigmoid, d_sigmoid)) nn.add_layer(DenseLayer(400, 150, sigmoid, d_sigmoid)) nn.add_layer(DenseLayer(150, 50, sigmoid, d_sigmoid)) nn.add_layer(DenseLayer(50, 10, same, d_same)) curtime = datetime.now() nn.train(dm.train_set, epochs=30, epoch_range=1) t = datetime.now() - curtime print('{} seconds for training'.format(t.seconds)) print('correct for {}% on train set'.format(nn.calculate_correct(dm.test_set))) results = [] for x, y in dm.test_set[:75]: res = nn.feed_forward(x) res = dm.classify(res)
self.batch_id[user_act] += test_batchsize batch_output = [] user_act_id = user_acts.index(user_act) batch_output += [user_act_id] * len(batch_data) return batch_data, batch_output, end_flag def generate_user_act_dataset(DialogData, user_act_data_ids): dialog_data = copy.deepcopy(DialogData) user_act_datas = {} for user_act, data_ids in user_act_data_ids.items(): if user_act in ['问询费用选项', '问询通话时长选项', '问询流量选项']: user_act = '问询说明' if user_act not in user_act_datas.keys(): user_act_datas[user_act] = [] for data_id in data_ids: user_act_datas[user_act] += dialog_data[data_id]["用户回复示例"] user_act_dataset = UserActDataset(user_act_datas) return user_act_dataset if __name__ == '__main__': data_manager = DataManager(os.path.join(BASE_DIR, '../../../data/tmp')) user_act_dataset = generate_user_act_dataset(data_manager.DialogData, user_act_data_ids) for user_act, data in user_act_dataset.data.items(): print(user_act) print(len(data))
class PassageFetcher(object): DATA_MANAGER = DataManager() LINE_CACHE_SIZE = 4 LOOK_FORWARD_SIZE = 4 TAG_LIMIT = 4 NEGATORY_QUOTE = "'Remember not only to say the right thing in the right place, but far more difficult still, to leave unsaid the wrong thing at the tempting moment.' - Benjamin Franklin" @staticmethod def FetchAndCompilePassages(tags): print(tags) tags = [str(tag) for tag in tags] # Validate tags valid_tags = PassageFetcher.ValidateTags(tags) print(valid_tags) feed = valid_tags if len( valid_tags) < PassageFetcher.TAG_LIMIT else valid_tags[ 0:PassageFetcher.TAG_LIMIT] authors = [] # Get passages result = "\n\n".join(PassageFetcher.FetchPassages(feed, authors)) if result == None or result.isspace(): return PassageFetcher.NEGATORY_QUOTE else: return tuple([result, feed, authors]) @staticmethod def FetchPassages(tags, authors): return [PassageFetcher.PassagePipe(tag, authors) for tag in tags] @staticmethod def PassagePipe(tag, authors): locations = PassageFetcher.GetLocations(tag) if locations == None: return "" coordinates = PassageFetcher.GetRandomLocation(locations) try: authors.append(Novels[coordinates[0]]) except Exception as e: print(e) raw_passage = PassageFetcher.ExtractPassage(coordinates) treated_passage = PassageFetcher.TreatPassage(raw_passage) return treated_passage @staticmethod def ValidateTags(tags): ### ADD THE SORT HERE ### valid_tags = [] for tag in tags: try: PassageFetcher.DATA_MANAGER.DATA[tag.lower()] valid_tags.append(tag) except: print(tag + " was not found.") return valid_tags @staticmethod def GetLocations(tag): try: locations = PassageFetcher.DATA_MANAGER.DATA[tag.lower()] except Exception as e: print("Error: ") return None return locations @staticmethod def GetRandomLocation(locations): return random.choice(locations) @staticmethod def ExtractPassage(coordinates): line_index = 0 word_index = 0 # Open up the book with open("./data/books/" + coordinates[0]) as file: # Compile regex sentence pattern line_cache = [] found_flag = False local_timer = PassageFetcher.LOOK_FORWARD_SIZE # Find line for line in file: words = line.split() # Find word for word in words: # If the word is found, compile the passage response if word_index == coordinates[2]: found_flag = True word_index += 1 line_index += 1 if line != '\r\n' and line != '\n' and line != '\r': line_cache.append(line) if len( line_cache ) > PassageFetcher.LINE_CACHE_SIZE and not found_flag: line_cache.pop(0) if found_flag: local_timer -= 1 if local_timer <= 0: break return "".join(line_cache) @staticmethod def TreatPassage(raw_passage): pattern = re.compile(r'([A-Z][^\.!?]*[\.!?])', re.M) sentences = pattern.findall(raw_passage) if len(sentences) > 3: sentences.pop(0) sentences.pop(len(sentences) - 1) return " ".join(sentences)