def __init__(self, sys_da_voc_json, usr_da_voc_json): self.sys_da_voc = json.load(open(sys_da_voc_json, encoding='utf8')) self.usr_da_voc = json.load(open(usr_da_voc_json, encoding='utf8')) self.database = Database() self.sys_da2id = {a: i for i, a in enumerate(self.sys_da_voc)} self.id2sys_da = {i: a for i, a in enumerate(self.sys_da_voc)} # 155 self.sys_da_dim = len(self.sys_da_voc) self.usr_da2id = {a: i for i, a in enumerate(self.usr_da_voc)} self.id2usr_da = {i: a for i, a in enumerate(self.usr_da_voc)} # 142 self.usr_da_dim = len(self.usr_da_voc) # 26 self.belief_state_dim = 0 # belief_state 中所有的 slot-values 的数量 for domain, svs in default_state()['belief_state'].items(): self.belief_state_dim += len(svs) self.db_res_dim = 4 self.state_dim = self.sys_da_dim + self.usr_da_dim + self.belief_state_dim + self.db_res_dim + 1 # terminated self.cur_domain = None self.belief_state = None self.db_res = None
def __init__(self): super(TradeDST, self).__init__() # load config common_config_path = os.path.join(get_config_path(), TradeDST.common_config_name) common_config = json.load(open(common_config_path)) model_config_path = os.path.join(get_config_path(), TradeDST.model_config_name) model_config = json.load(open(model_config_path)) model_config.update(common_config) self.model_config = model_config self.model_config['data_path'] = os.path.join( get_data_path(), 'crosswoz/dst_trade_data') self.model_config['n_gpus'] = 0 if self.model_config[ 'device'] == 'cpu' else torch.cuda.device_count() self.model_config['device'] = torch.device(self.model_config['device']) if model_config['load_embedding']: model_config['hidden_size'] = 300 # download data for model_key, url in TradeDST.model_urls.items(): dst = os.path.join(self.model_config['data_path'], model_key) if model_key.endswith('pth'): file_name = 'trained_model_path' elif model_key.endswith('pkl'): file_name = model_key.rsplit('-', maxsplit=1)[0] else: file_name = model_key.split('.')[0] # ontology self.model_config[file_name] = dst if not os.path.exists(dst) or not self.model_config['use_cache']: download_from_url(url, dst) # load date & model ontology = json.load( open(self.model_config['ontology'], 'r', encoding='utf8')) self.all_slots = get_slot_information(ontology) self.gate2id = {'ptr': 0, 'none': 1} self.id2gate = {id_: gate for gate, id_ in self.gate2id.items()} self.lang = pickle.load(open(self.model_config['lang'], 'rb')) self.mem_lang = pickle.load(open(self.model_config['mem-lang'], 'rb')) model = Trade( lang=self.lang, vocab_size=len(self.lang.index2word), hidden_size=self.model_config['hidden_size'], dropout=self.model_config['dropout'], num_encoder_layers=self.model_config['num_encoder_layers'], num_decoder_layers=self.model_config['num_decoder_layers'], pad_id=self.model_config['pad_id'], slots=self.all_slots, num_gates=len(self.gate2id), unk_mask=self.model_config['unk_mask']) model.load_state_dict( torch.load(self.model_config['trained_model_path'])) self.model = model.to(self.model_config['device']).eval() print(f'>>> {self.model_config["trained_model_path"]} loaded ...') self.state = default_state() print('>>> State initialized ...')
def init_session(self, state: Optional[dict] = None) -> None: """Initialize ``self.state`` with a default state. Args: state: see xbot.util.state.default_state """ self.dialogue_state = default_state() if not state else deepcopy(state)
def __init__(self): super(FSMDST, self).__init__() self.dialogue_state = default_state() self.dialogue_state["cur_domain"] = "餐馆" self.restaurant_states = list( self.dialogue_state["belief_state"]["餐馆"].keys()) self.restaurant_states.append("Request") self.machine = Machine(model=self, states=self.restaurant_states, initial="greet") self.dest2trigger = { "推荐菜": { "trigger": "inform_restaurant_dish", "callback": "set_recommend_dish", }, "评分": { "trigger": "inform_restaurant_rating", "callback": "set_rating" }, "人均消费": { "trigger": "inform_restaurant_avg_cost", "callback": "set_avg_cost", }, "周边酒店": { "trigger": "inform_restaurant_surrounding_hotel", "callback": "set_surrounding_hotel", }, "周边景点": { "trigger": "inform_restaurant_surrounding_attraction", "callback": "set_surrounding_attraction", }, "周边餐馆": { "trigger": "inform_restaurant_surrounding_restaurant", "callback": "set_surrounding_restaurant", }, } for slot, trigger in self.dest2trigger.items(): self.machine.add_transition( trigger=trigger["trigger"], source="*", dest=slot, before=trigger["callback"], )
def __init__(self): super(FSMDST, self).__init__() self.dialogue_state = default_state() self.dialogue_state['cur_domain'] = '餐馆' self.restaurant_states = list( self.dialogue_state['belief_state']['餐馆'].keys()) self.restaurant_states.append('Request') self.machine = Machine(model=self, states=self.restaurant_states, initial='greet') self.dest2trigger = { '推荐菜': { 'trigger': 'inform_restaurant_dish', 'callback': 'set_recommend_dish' }, '评分': { 'trigger': 'inform_restaurant_rating', 'callback': 'set_rating' }, '人均消费': { 'trigger': 'inform_restaurant_avg_cost', 'callback': 'set_avg_cost' }, '周边酒店': { 'trigger': 'inform_restaurant_surrounding_hotel', 'callback': 'set_surrounding_hotel' }, '周边景点': { "trigger": 'inform_restaurant_surrounding_attraction', 'callback': 'set_surrounding_attraction' }, '周边餐馆': { 'trigger': 'inform_restaurant_surrounding_restaurant', 'callback': 'set_surrounding_restaurant' } } for slot, trigger in self.dest2trigger.items(): self.machine.add_transition(trigger=trigger['trigger'], source='*', dest=slot, before=trigger['callback'])
def __init__(self): super(BertDST, self).__init__() # load config infer_config = self.load_config() # download data self.download_data(infer_config) self.ontology = json.load( open(infer_config['cleaned_ontology'], 'r', encoding='utf8')) self.model = BertForSequenceClassification.from_pretrained( infer_config['model_dir']) self.model.to(infer_config['device']) self.tokenizer = BertTokenizer.from_pretrained( infer_config['model_dir']) self.config = infer_config self.model.eval() self.state = default_state() self.domains = set(self.state['belief_state'].keys())
def init_session(self): self.state = default_state()
if absence: return False s_arr = [val for val in plan_values.split(" ") if val] # 只要有一个推荐菜没有被包含就视为不满足约束 if [val for val in s_arr if contains(db_values, val)]: return False else: plan_values = options["params"] if plan_values is not None: if absence: return False if db_values.find(plan_values) < 0: return False return True return [item for item in db if func3(item)] if __name__ == "__main__": from pprint import pprint from collections import Counter database = Database() state = default_state() dishes = {} for n, v in database.query(state["belief_state"], "餐馆"): for dish in v["推荐菜"]: dishes.setdefault(dish, 0) dishes[dish] += 1 pprint(Counter(dishes))
def update(self, usr_da=None): """update belief_state, cur_domain, request_slot :param usr_da: List[List[intent, domain, slot, value]] :return: state """ sys_da = self.state['system_action'] # 统计各个意图下的 domain select_domains = Counter([x[1] for x in usr_da if x[0] == 'Select']) request_domains = Counter([x[1] for x in usr_da if x[0] == 'Request']) inform_domains = Counter([x[1] for x in usr_da if x[0] == 'Inform']) sys_domains = Counter( [x[1] for x in sys_da if x[0] in ['Inform', 'Recommend']]) # 为什么首选 select_domain # 观察数据集可以发现,Select 意图出现的时候是主导整句话的,即便出现了 # Inform 和 Request 也是为了为 Select 提供辅助信息,Inform 排在 Request 之后,也是这个道理 if len(select_domains) > 0: self.state['cur_domain'] = select_domains.most_common(1)[0][0] elif len(request_domains) > 0: self.state['cur_domain'] = request_domains.most_common(1)[0][0] elif len(inform_domains) > 0: self.state['cur_domain'] = inform_domains.most_common(1)[0][0] elif len(sys_domains) > 0: self.state['cur_domain'] = sys_domains.most_common(1)[0][0] else: self.state['cur_domain'] = None # 当 system action 中没有 inform 的意图且存在 NoOffer 的意图,判定为确实没 offer # 要满足没有 Inform,是因为如果存在 Inform,意味着 system 基于之前的约束信息可能提出了新的考虑意见, # 所以只有当 system 基于当前约束完全无法给出建议的时候,当前 domain 的约束才算失效 no_offer = 'NoOffer' in [x[0] for x in sys_da ] and 'Inform' not in [x[0] for x in sys_da] # DONE: clean cur domain constraints because no offer if no_offer: if self.state[ 'cur_domain']: # 没有 offer 则清空对应 domain 的 state,即 slot 对应的 value self.state['belief_state'][ self.state['cur_domain']] = deepcopy( default_state()['belief_state'][ self.state['cur_domain']]) # DONE: clean request slot # 上一个 system action 中的 inform 和 recommend 表示已经完成的 request,所以从 request_slots 移除 for domain, slot in deepcopy(self.state['request_slots']): if [domain, slot] in [ x[1:3] for x in sys_da if x[0] in ['Inform', 'Recommend'] ]: self.state['request_slots'].remove([domain, slot]) # DONE: domain switch for intent, domain, slot, value in usr_da: # ["Select", "酒店", "源领域", "餐馆"] 找餐馆附近的酒店,所以餐馆是 from_domain if intent == 'Select': from_domain = value name = self.state['belief_state'][from_domain]['名称'] if name: # 如果当前 domain 等于前一个 domain 那么信息清零再更新, # 与源领域一致说明同样的 domain 要换个具体值了,所以前面的信息没用了 if domain == from_domain: self.state['belief_state'][domain] = deepcopy( default_state()['belief_state'][domain]) self.state['belief_state'][domain]['周边{}'.format( from_domain)] = name for intent, domain, slot, value in usr_da: if intent == 'Inform': if slot in [ '名称', '游玩时间', '酒店类型', '出发地', '目的地', '评分', '门票', '价格', '人均消费' ]: self.state['belief_state'][domain][slot] = value elif slot == '推荐菜': # 推荐菜可以有多个 if not self.state['belief_state'][domain][slot]: self.state['belief_state'][domain][slot] = value else: self.state['belief_state'][domain][slot] += ' ' + value elif '酒店设施' in slot: # ["Inform", "酒店", "酒店设施-吹风机", "是"] if value == '是': facility = slot.split('-')[1] if not self.state['belief_state'][domain]['酒店设施']: self.state['belief_state'][domain][ '酒店设施'] = facility else: self.state['belief_state'][domain][ '酒店设施'] += ' ' + facility elif intent == 'Request': # 存入新增的 domain-slot self.state['request_slots'].append([domain, slot]) return self.state
def init_session(self, state=None): """Initialize ``self.state`` with a default state. :state: see xbot.util.state.default_state """ self.state = default_state() if not state else deepcopy(state)
def __init__(self): super(RuleDST, self).__init__() self.state = default_state()
def __init__(self): super(TradeDST, self).__init__() # load config common_config_path = os.path.join(get_config_path(), TradeDST.common_config_name) common_config = json.load(open(common_config_path)) model_config_path = os.path.join(get_config_path(), TradeDST.model_config_name) model_config = json.load(open(model_config_path)) model_config.update(common_config) self.model_config = model_config self.model_config["data_path"] = os.path.join( get_data_path(), "crosswoz/dst_trade_data") self.model_config["n_gpus"] = (0 if self.model_config["device"] == "cpu" else torch.cuda.device_count()) self.model_config["device"] = torch.device(self.model_config["device"]) if model_config["load_embedding"]: model_config["hidden_size"] = 300 # download data for model_key, url in TradeDST.model_urls.items(): dst = os.path.join(self.model_config["data_path"], model_key) if model_key.endswith("pth"): file_name = "trained_model_path" elif model_key.endswith("pkl"): file_name = model_key.rsplit("-", maxsplit=1)[0] else: file_name = model_key.split(".")[0] # ontology self.model_config[file_name] = dst if not os.path.exists(dst) or not self.model_config["use_cache"]: download_from_url(url, dst) # load date & model ontology = json.load( open(self.model_config["ontology"], "r", encoding="utf8")) self.all_slots = get_slot_information(ontology) self.gate2id = {"ptr": 0, "none": 1} self.id2gate = {id_: gate for gate, id_ in self.gate2id.items()} self.lang = pickle.load(open(self.model_config["lang"], "rb")) self.mem_lang = pickle.load(open(self.model_config["mem-lang"], "rb")) model = Trade( lang=self.lang, vocab_size=len(self.lang.index2word), hidden_size=self.model_config["hidden_size"], dropout=self.model_config["dropout"], num_encoder_layers=self.model_config["num_encoder_layers"], num_decoder_layers=self.model_config["num_decoder_layers"], pad_id=self.model_config["pad_id"], slots=self.all_slots, num_gates=len(self.gate2id), unk_mask=self.model_config["unk_mask"], ) model.load_state_dict( torch.load(self.model_config["trained_model_path"])) self.model = model.to(self.model_config["device"]).eval() print(f'>>> {self.model_config["trained_model_path"]} loaded ...') self.state = default_state() print(">>> State initialized ...")
def update(self, usr_da=None): """update belief_state, cur_domain, request_slot :param usr_da: List[List[intent, domain, slot, value]] :return: state """ sys_da = self.state["system_action"] # 统计各个意图下的 domain select_domains = Counter([x[1] for x in usr_da if x[0] == "Select"]) request_domains = Counter([x[1] for x in usr_da if x[0] == "Request"]) inform_domains = Counter([x[1] for x in usr_da if x[0] == "Inform"]) sys_domains = Counter( [x[1] for x in sys_da if x[0] in ["Inform", "Recommend"]]) # 确定domain # 为什么首选 select_domain # 观察数据集可以发现,Select 意图出现的时候是主导整句话的,即便出现了 # Inform 和 Request 也是为了为 Select 提供辅助信息,Inform 排在 Request 之后,也是这个道理 if len(select_domains) > 0: self.state["cur_domain"] = select_domains.most_common(1)[0][0] elif len(request_domains) > 0: self.state["cur_domain"] = request_domains.most_common(1)[0][0] elif len(inform_domains) > 0: self.state["cur_domain"] = inform_domains.most_common(1)[0][0] elif len(sys_domains) > 0: self.state["cur_domain"] = sys_domains.most_common(1)[0][0] else: self.state["cur_domain"] = None # 当 system action 中没有 inform 的意图且存在 NoOffer 的意图,判定为确实没 offer # 要满足没有 Inform,是因为如果存在 Inform,意味着 system 基于之前的约束信息可能提出了新的考虑意见, # 所以只有当 system 基于当前约束完全无法给出建议的时候,当前 domain 的约束才算失效 no_offer = "NoOffer" in [x[0] for x in sys_da ] and "Inform" not in [x[0] for x in sys_da] # DONE: clean cur domain constraints because no offer if no_offer: # ISSUE: 过于暴力的做法 if self.state[ "cur_domain"]: # 没有 offer 则清空对应 domain 的 state,即 slot 对应的 value self.state["belief_state"][ self.state["cur_domain"]] = deepcopy( default_state()["belief_state"][ self.state["cur_domain"]]) # DONE: clean request slot # 上一个 system action 中的 inform 和 recommend 表示已经完成的 request,所以从 request_slots 移除 for domain, slot in deepcopy(self.state["request_slots"]): # 已解决的slot全部清除过于暴力 if [domain, slot] in [ x[1:3] for x in sys_da if x[0] in ["Inform", "Recommend"] ]: self.state["request_slots"].remove([domain, slot]) # DONE: domain switch # 非常强依赖前面的NLU for intent, domain, slot, value in usr_da: # ["Select", "酒店", "源领域", "餐馆"] 找餐馆附近的酒店,所以餐馆是 from_domain if intent == "Select": from_domain = value name = self.state["belief_state"][from_domain]["名称"] if name: # 如果当前 domain 等于前一个 domain 那么信息清零再更新, # 与源领域一致说明同样的 domain 要换个具体值了,所以前面的信息没用了 if domain == from_domain: self.state["belief_state"][domain] = deepcopy( default_state()["belief_state"][domain]) self.state["belief_state"][domain]["周边{}".format( from_domain)] = name for intent, domain, slot, value in usr_da: if intent == "Inform": if slot in [ "名称", "游玩时间", "酒店类型", "出发地", "目的地", "评分", "门票", "价格", "人均消费", ]: self.state["belief_state"][domain][slot] = value elif slot == "推荐菜": # 推荐菜可以有多个 if not self.state["belief_state"][domain][slot]: self.state["belief_state"][domain][slot] = value else: self.state["belief_state"][domain][slot] += " " + value elif "酒店设施" in slot: # ["Inform", "酒店", "酒店设施-吹风机", "是"] if value == "是": facility = slot.split("-")[1] if not self.state["belief_state"][domain]["酒店设施"]: self.state["belief_state"][domain][ "酒店设施"] = facility else: self.state["belief_state"][domain][ "酒店设施"] += " " + facility elif intent == "Request": # 存入新增的 domain-slot self.state["request_slots"].append([domain, slot]) # ISSUE: 词槽澄清 # ISSUE: Repeat Intent # ISSUE: System State # ISSUE: Hidden Slot # ISSUE: 平级槽和依赖槽 return self.state
def init_session(self) -> None: """Initiate state of one session. """ self.state = default_state()