def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/mle_policy_camrest.zip' ): root_dir = os.path.dirname( os.path.dirname( os.path.dirname( os.path.dirname(os.path.dirname( os.path.abspath(__file__)))))) with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f: cfg = json.load(f) voc_file = os.path.join(root_dir, 'data/camrest/sys_da_voc.txt') voc_opp_file = os.path.join(root_dir, 'data/camrest/usr_da_voc.txt') self.vector = CamrestVector(voc_file, voc_opp_file) self.policy = MultiDiscretePolicy(self.vector.state_dim, cfg['h_dim'], self.vector.da_dim).to(device=DEVICE) if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MLE Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.load(cfg['load'])
def __init__(self, mode): """ SVM NLU initialization. Args: mode (str): can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on. Example: nlu = SVMNLU(mode='usr') """ assert mode == 'usr' or mode == 'sys' or mode == 'all' model_file = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/svm_camresst_{}.zip'.format( mode) config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/camrest_{}.cfg'.format(mode)) self.config = configparser.ConfigParser() self.config.read(config_file) self.c = Classifier.classifier(self.config) model_path = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), self.config.get("train", "output")) model_dir = os.path.dirname(model_path) if not os.path.exists(model_path): if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) archive.close() self.c.load(model_path)
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/vhus_simulator_camrest.zip' ): with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f: config = json.load(f) manager = UserDataManager() voc_goal_size, voc_usr_size, voc_sys_size = manager.get_voc_size() self.user = VHUS(config, voc_goal_size, voc_usr_size, voc_sys_size).to(device=DEVICE) self.goal_gen = GoalGenerator() self.manager = manager self.user.eval() if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for VHUS Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_simulator.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.load(config['load'])
def __init__(self, model_file=DEFAULT_MODEL_URL, name='Sequicity'): """ Sequicity initialization Args: model_file (str): trained model path or url. default="https://tatk-data.s3-ap-northeast-1.amazonaws.com/sequicity_multiwoz.zip" Example: sequicity = Sequicity() """ super(Sequicity, self).__init__(name=name) config_file = DEFAULT_CONFIG_FILE c = json.load(open(config_file)) cfg.init_handler(c['tsdf_init']) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')): print('down load data from', DEFAULT_ARCHIVE_FILE_URL) archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', os.path.join(DEFAULT_DIRECTORY,'multiwoz/')) archive.extractall(os.path.join(DEFAULT_DIRECTORY,'multiwoz/')) archive.close() model_path = os.path.join(DEFAULT_DIRECTORY,c['tsdf_init']['model_path']) if not os.path.exists(model_path): model_dir = os.path.dirname(model_path) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') print('down load data from', model_file) archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', model_dir) archive.extractall(model_dir) archive.close() torch.manual_seed(cfg.seed) torch.cuda.manual_seed(cfg.seed) random.seed(cfg.seed) np.random.seed(cfg.seed) self.m = Model('multiwoz') self.m.count_params() self.m.load_model() self.init_session()
def __init__(self, mode, model_file): """ BERT NLU initialization. Args: mode (str): can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on. model_file (str): trained model path or url, should be coherent with mode. Example: nlu = BERTNLU(mode='usr', model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_camrest_usr.zip') """ assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/camrest_{}.json'.format(mode)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'data.pkl')): preprocess(mode) data = pickle.load(open(os.path.join(data_dir, 'data.pkl'), 'rb')) intent_vocab = pickle.load(open(os.path.join(data_dir, 'intent_vocab.pkl'), 'rb')) tag_vocab = pickle.load(open(os.path.join(data_dir, 'tag_vocab.pkl'), 'rb')) dataloader = Dataloader(data, intent_vocab, tag_vocab, config['model']["pre-trained"]) best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) checkpoint = torch.load(best_model_path, map_location=DEVICE) print('train step', checkpoint['step']) model = BertNLU(config['model'], dataloader.intent_dim, dataloader.tag_dim, DEVICE=DEVICE, intent_weight=dataloader.intent_weight) model_dict = model.state_dict() state_dict = {k: v for k, v in checkpoint['model_state_dict'].items() if k in model_dict.keys()} model_dict.update(state_dict) model.load_state_dict(model_dict) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
def __init__(self, mode, config_file, model_file): """ BERT NLU initialization. Args: mode (str): can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on. model_file (str): model path or url Example: nlu = BERTNLU(mode='all', model_file='https://convlab.blob.core.windows.net/models/bert_multiwoz_all_context.zip') """ assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load(open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader(intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) bert_config = BertConfig.from_pretrained(config['model']['pretrained_weights']) best_model_path = os.path.join(output_dir, 'pytorch_model.bin') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict(torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
def auto_download(self): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.config_path, 'model/rnn_model_state_dict.th')) and \ os.path.exists(os.path.join(self.config_path, 'selection_model_state_dict.th')): return models_dir = os.path.join(self.config_path, 'models') cached_path(self.file_url, models_dir) files = os.listdir(models_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print( 'allennlp download file error: RnnRollout Deal_or_Not data download failed.' ) raise e zip_file_path = os.path.join(models_dir, target_file + '.zip') shutil.copyfile(os.path.join(models_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(models_dir)
def auto_download(self): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.data_dir, 'models')) and \ os.path.exists(os.path.join(self.data_dir, 'data')) and \ os.path.exists(os.path.join(self.data_dir, 'word-vectors')): return cached_path(self.file_url, self.data_dir) files = os.listdir(self.data_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print( 'allennlp download file error: MDBT Multiwoz data download failed.' ) raise e zip_file_path = os.path.join(self.data_dir, target_file + '.zip') shutil.copyfile(os.path.join(self.data_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(self.data_dir)
def load(self, archive_file, model_file, filename): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MLE Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) policy_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_mle.pol.mdl') if os.path.exists(policy_mdl): self.policy.load_state_dict(torch.load(policy_mdl)) print('<<dialog policy>> loaded checkpoint from file: {}'.format(policy_mdl))
def __init__(self, mode, config_file, model_file): assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load( open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader( intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) bert_config = BertConfig.from_pretrained( config['model']['pretrained_weights']) best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
def auto_download(): model_path = os.path.join(os.path.dirname(__file__), os.pardir, 'model') data_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data') db_path = os.path.join(os.path.dirname(__file__), os.pardir, 'db') root_path = os.path.join(os.path.dirname(__file__), os.pardir) urls = {model_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_model.zip', data_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_data.zip', db_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_db.zip'} for path in [model_path, data_path, db_path]: if not os.path.exists(path): file_url = urls[path] print('Downloading from: ', file_url) archive_file = cached_path(file_url) print('Extracting...') archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_path)
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, use_cuda=False, is_user=False, model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/nlg_sclstm_multiwoz.zip' ): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for SC-LSTM is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'resource')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.USE_CUDA = use_cuda self.args, self.config = parse(is_user) self.dataset = SimpleDatasetWoz(self.config) # get model hyper-parameters hidden_size = self.config.getint('MODEL', 'hidden_size') # get feat size d_size = self.dataset.do_size + self.dataset.da_size + self.dataset.sv_size # len of 1-hot feat vocab_size = len(self.dataset.word2index) self.model = LMDeep('sclstm', vocab_size, vocab_size, hidden_size, d_size, n_layer=self.args['n_layer'], use_cuda=use_cuda) model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.args['model_path']) # print(model_path) assert os.path.isfile(model_path) self.model.load_state_dict(torch.load(model_path)) self.model.eval() if use_cuda: self.model.cuda()
tag_vocab = pickle.load(open(os.path.join(data_dir, 'tag_vocab.pkl'), 'rb')) for key in data: print('{} set size: {}'.format(key, len(data[key]))) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) dataloader = Dataloader(data, intent_vocab, tag_vocab, config['model']["pre-trained"]) best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from zipped_model_path param') archive_file = cached_path( os.path.join(root_dir, config['zipped_model_path'])) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) checkpoint = torch.load(best_model_path, map_location=DEVICE) print('best_intent_step', checkpoint['best_intent_step']) print('best_tag_step', checkpoint['best_tag_step']) model = BertNLU(config['model'], dataloader.intent_dim, dataloader.tag_dim, DEVICE=DEVICE, intent_weight=dataloader.intent_weight) model_dict = model.state_dict() state_dict = {
'rb')) for key in data: print('{} set size: {}'.format(key, len(data[key]))) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) dataloader = Dataloader(data, intent_vocab, tag_vocab, config['model']["pre-trained"]) best_model_path = best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from zipped_model_path param') archive_file = cached_path(config['zipped_model_path']) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall() archive.close() print('Load from', best_model_path) checkpoint = torch.load(best_model_path, map_location=DEVICE) print('train step', checkpoint['step']) model = BertNLU(config['model'], dataloader.intent_dim, dataloader.tag_dim, DEVICE=DEVICE, intent_weight=dataloader.intent_weight) model_dict = model.state_dict() state_dict = { k: v