def __init__(self, ): self.frames = read_zipped_json( os.path.join(DATA_ROOT, 'frames/Ori', 'train.json.zip'), 'train.json') frames_frames_domain_slot_map = { # ('frame', 'category'): ('hotel', 'category'), ('frame', 'dst_city'): ('hotel', 'location'), # ('frame', 'gst_rating'): ('hotel', 'gst_rating'), ('frame', 'name'): ('hotel', 'name'), ('frame', 'or_city'): ('trip', 'or_city'), # ('frame', 'seat'): ('trip', 'seat'), } frames_sgd_domain_slot_map = { ('frame', 'dst_city'): ('hotels', 'dst_city'), ('frame', 'name'): ('hotels', 'hotel_name'), ('frame', 'or_city'): ('travel', 'location'), } frames_db_dir = os.path.join( get_root_path(), "convlab2/laug/Word_Perturbation/db/frames-db/") sgd_db_dir = os.path.join( get_root_path(), "convlab2/laug/Word_Perturbation/db/sgd-db/") loader_args = [ MultiSourceDBLoaderArgs(frames_db_dir, frames_frames_domain_slot_map), MultiSourceDBLoaderArgs(sgd_db_dir, frames_sgd_domain_slot_map) ] self.db_loader = MultiSourceDBLoader(loader_args)
def __init__(self): self.max_label_length = 32 self.max_turn_length = 22 self.hidden_dim = 100 self.num_rnn_layers = 1 self.zero_init_rnn = False self.attn_head = 4 self.do_eval = True self.do_train = False self.distance_metric = 'cosine' self.train_batch_size = 1 self.dev_batch_size = 1 self.eval_batch_size = 16 self.learning_rate = 5e-5 self.num_train_epochs = 3 self.patience = 10 self.warmup_proportion = 0.1 self.local_rank = -1 self.seed = 42 self.gradient_accumulation_steps = 4 self.fp16 = False self.loss_scale = 0 self.do_not_use_tensorboard = False self.fix_utterance_encoder = False self.do_eval = True self.num_train_epochs = 300 self.bert_model = os.path.join( convlab2.get_root_path(), "pre-trained-models/bert-base-multilingual-uncased") self.bert_model_cache_dir = os.path.join(convlab2.get_root_path(), "pre-trained-models/") self.bert_model_name = "bert-base-multilingual-uncased" self.do_lower_case = True self.task_name = 'bert-gru-sumbt' self.nbt = 'rnn' # self.output_dir = os.path.join(path, 'ckpt/') self.target_slot = 'all' self.learning_rate = 5e-5 self.train_batch_size = 1 self.eval_batch_size = 16 self.distance_metric = 'euclidean' self.patience = 15 self.hidden_dim = 300 self.max_label_length = 32 self.max_seq_length = 64 self.max_turn_length = 22 self.fp16_loss_scale = 0.0 self.data_dir = 'data/multiwoz/' self.tf_dir = 'tensorboard' self.tmp_data_dir = 'processed_data/' self.output_dir = 'model_output_mbert_ft/'
def __init__(self, goal_model_path=os.path.join( get_root_path(), 'data/multiwoz/goal/new_goal_model.pkl'), corpus_path=None, boldify=False, sample_info_from_trainset=True, sample_reqt_from_trainset=False): """ Args: goal_model_path: path to a goal model corpus_path: path to a dialog corpus to build a goal model boldify: highlight some information in the goal message sample_info_from_trainset: if True, sample info slots combination from train set, else sample each slot independently sample_reqt_from_trainset: if True, sample reqt slots combination from train set, else sample each slot independently """ self.goal_model_path = goal_model_path self.corpus_path = corpus_path self.db = Database() self.boldify = do_boldify if boldify else null_boldify self.sample_info_from_trainset = sample_info_from_trainset self.sample_reqt_from_trainset = sample_reqt_from_trainset self.train_database = self.db.query('train', []) if os.path.exists(self.goal_model_path): self.ind_slot_dist, self.ind_slot_value_dist, self.domain_ordering_dist, self.book_dist, self.slots_num_dist, self.slots_combination_dist = pickle.load( open(self.goal_model_path, 'rb')) print('Loading goal model is done') else: self._build_goal_model() print('Building goal model is done')
def __init__(self, name, args, sel_args, train=False, diverse=False, max_total_len=100, model_url='https://tatk-data.s3-ap-northeast-1.amazonaws.com/rnnrollout_dealornot.zip'): self.config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs') self.file_url = model_url self.auto_download() if not os.path.exists(self.config_path): os.mkdir(self.config_path) _model_path = os.path.join(self.config_path, 'models') self.model_path = _model_path if not os.path.exists(_model_path): os.makedirs(_model_path) self.data_path = os.path.join(get_root_path(), args.data) domain = get_domain(args.domain) corpus = RnnModel.corpus_ty(domain, self.data_path, freq_cutoff=args.unk_threshold, verbose=True, sep_sel=args.sep_sel) model = RnnModel(corpus.word_dict, corpus.item_dict_old, corpus.context_dict, corpus.count_dict, args) state_dict = utils.load_model(os.path.join(self.config_path, args.model_file)) # RnnModel model.load_state_dict(state_dict) sel_model = SelectionModel(corpus.word_dict, corpus.item_dict_old, corpus.context_dict, corpus.count_dict, sel_args) sel_state_dict = utils.load_model(os.path.join(self.config_path, sel_args.selection_model_file)) sel_model.load_state_dict(sel_state_dict) super(DealornotAgent, self).__init__(model, sel_model, args, name, train, diverse, max_total_len) self.vis = args.visual
def __init__(self, ): self.multiwoz = read_zipped_json( os.path.join(DATA_ROOT, 'multiwoz', 'train.json.zip'), 'train.json') multiwoz_db_dir = os.path.join(DATA_ROOT, 'multiwoz', 'db') multiwoz_multiwoz_domain_slot_map = { ('attraction', 'area'): ('attraction', 'Area'), ('attraction', 'type'): ('attraction', 'Type'), ('attraction', 'name'): ('attraction', 'Name'), ('attraction', 'address'): ('attraction', 'Addr'), ('hospital', 'department'): ('hospital', 'Department'), ('hospital', 'address'): ('hospital', 'Addr'), ('hotel', 'type'): ('hotel', 'Type'), ('hotel', 'area'): ('hotel', 'Area'), ('hotel', 'name'): ('hotel', 'Name'), ('hotel', 'address'): ('hotel', 'Addr'), ('restaurant', 'food'): ('restaurant', 'Food'), ('restaurant', 'area'): ('restaurant', 'Area'), ('restaurant', 'name'): ('restaurant', 'Name'), ('restaurant', 'address'): ('restaurant', 'Addr'), ('train', 'destination'): ('train', 'Dest'), ('train', 'departure'): ('train', 'Depart') } multiwoz_sgd_domain_slot_map = { ('train', 'dest'): ('train', 'to'), ('train', 'depart'): ('train', 'from'), ('hotel', 'name'): ('hotels', 'hotel_name'), ('hotel', 'addr'): ('hotels', 'address'), ('attraction', 'name'): ('travel', 'attraction_name'), ('restaurant', 'name'): ('restaurants', 'restaurant_name'), ('restaurant', 'addr'): ('restaurants', 'street_address') } loader_args = [ MultiSourceDBLoaderArgs(multiwoz_db_dir, multiwoz_multiwoz_domain_slot_map) ] sgd_db_dir = os.path.join( get_root_path(), "convlab2/laug/Word_Perturbation/db/sgd-db/") loader_args.append( MultiSourceDBLoaderArgs(sgd_db_dir, multiwoz_sgd_domain_slot_map)) self.db_loader = MultiSourceDBLoader(loader_args)
user_goal[dom]['fail_book'].items())))[0][0][0] if adjusted_slot in ['internet', 'parking']: message.append( templates[dom]['fail_book ' + adjusted_slot + ' ' + user_goal[dom]['book'][adjusted_slot]]) else: message.append( templates[dom]['fail_book ' + adjusted_slot].format( self.boldify( user_goal[dom]['book'][adjusted_slot]))) dm = message[mess_ptr4domain:] mess_ptr4domain = len(message) message_by_domain.append(' '.join(dm)) if boldify == do_boldify: for i, m in enumerate(message): message[i] = message[i].replace('wifi', "<b>wifi</b>") message[i] = message[i].replace('internet', "<b>internet</b>") message[i] = message[i].replace('parking', "<b>parking</b>") return message, message_by_domain if __name__ == '__main__': goal_generator = GoalGenerator(corpus_path=os.path.join( get_root_path(), 'data/multiwoz/train.json'), sample_reqt_from_trainset=True) pprint(goal_generator.get_user_goal())
# fail_book if 'fail_book' in user_goal[dom]: adjusted_slot = list(filter(lambda x: x[0][1] != x[1][1], zip(user_goal[dom]['book'].items(), user_goal[dom]['fail_book'].items())))[0][ 0][0] if adjusted_slot in ['internet', 'parking']: message.append( templates[dom]['fail_book ' + adjusted_slot + ' ' + user_goal[dom]['book'][adjusted_slot]]) else: message.append(templates[dom]['fail_book ' + adjusted_slot].format( self.boldify(user_goal[dom]['book'][adjusted_slot]))) dm = message[mess_ptr4domain:] mess_ptr4domain = len(message) message_by_domain.append(' '.join(dm)) if boldify == do_boldify: for i, m in enumerate(message): message[i] = message[i].replace('wifi', "<b>wifi</b>") message[i] = message[i].replace('internet', "<b>internet</b>") message[i] = message[i].replace('parking', "<b>parking</b>") return message, message_by_domain if __name__ == '__main__': goal_generator = GoalGenerator(corpus_path=os.path.join(get_root_path(), 'data/multiwoz/train.json'), sample_reqt_from_trainset=True) # goal_generator._build_goal_model() pprint(goal_generator.get_user_goal())
def get_context_generator(context_file): return utils.ContextGenerator(os.path.join(get_root_path(), context_file))