def download_data( self, data_url="https://convlab.blob.core.windows.net/convlab-2/trade_crosswoz_data.zip" ): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.crosswoz_root, 'data/crosswoz')) and \ os.path.exists(os.path.join(self.crosswoz_root, 'data/dev_dials.json')): return data_dir = os.path.join(self.crosswoz_root, 'data') if not os.path.exists(data_dir): os.mkdir(data_dir) zip_file_path = os.path.join(data_dir, 'trade_crosswoz_data.zip') if not os.path.exists(os.path.join(data_dir, 'trade_crosswoz_data.zip')): print('downloading crosswoz TRADE data files...') cached_path(data_url, data_dir) files = os.listdir(data_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print( 'allennlp download file error: TRADE Cross model download failed.' ) raise e shutil.copyfile(os.path.join(data_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: print('unzipping data file ...') zip_ref.extractall(data_dir)
def download_data(self, data_url="https://convlab.blob.core.windows.net/convlab-2/comer.zip"): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.multiwoz_root, 'data/mwoz2_dm.dict')) and \ os.path.exists(os.path.join(self.multiwoz_root, 'data/mwoz2_sl.dict')) and \ os.path.exists(os.path.join(self.multiwoz_root, 'data/save_data.tgt.dict')): return print(os.path.join(self.multiwoz_root, 'data/mwoz2_dm.dict')) print(os.path.join(self.multiwoz_root, 'data/mwoz2_sl.dict')) print(os.path.join(self.multiwoz_root, 'data/save_data.tgt.dict')) data_dir = os.path.join(self.multiwoz_root, 'data') if not os.path.exists(data_dir): os.mkdir(data_dir) zip_file_path = os.path.join(data_dir, 'comer.zip') if not os.path.exists(os.path.join(data_dir, 'comer.zip')): print('downloading multiwoz COMER data files...') cached_path(data_url, data_dir) files = os.listdir(data_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print('allennlp download file error: COMER model download failed.') raise e shutil.copyfile(os.path.join(data_dir, target_file), zip_file_path) print("Unzipping comer.zip...") with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(data_dir) print("Moving files and dirs...") os.system("mv data/data bak/") os.system("rm -rf data/") os.system("mv bak/ data/")
def download_model( self, model_url="https://convlab.blob.core.windows.net/convlab-2/trade_multiwoz_model.zip" ): """Automatically download the pretrained model and necessary data.""" if os.path.exists( os.path.join(self.multiwoz_root, 'model/TRADE-multiwozdst')): return model_dir = os.path.join(self.multiwoz_root, 'model') if not os.path.exists(model_dir): os.mkdir(model_dir) zip_file_path = os.path.join(model_dir, 'trade_multiwoz_model.zip') if not os.path.exists( os.path.join(model_dir, 'trade_multiwoz_model.zip')): print('downloading multiwoz Trade model files...') cached_path(model_url, model_dir) files = os.listdir(model_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print( 'allennlp download file error: TRADE Cross model download failed.' ) raise e shutil.copyfile(os.path.join(model_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: print('unzipping model file ...') zip_ref.extractall(model_dir)
def __init__(self, model_file=DEFAULT_MODEL_URL, name='DAMD'): """ Sequicity initialization Args: model_file (str): trained model path or url. Example: damd = Damd() """ super(Damd, self).__init__(name=name) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/data')): print('down load data from', DEFAULT_ARCHIVE_FILE_URL) archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'multiwoz/')) archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/')) archive.close() model_path = os.path.join(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'), cfg.eval_load_path) if not os.path.exists(model_path): model_dir = os.path.dirname(model_path) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') print('down load data from', model_file) archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', model_dir) archive.extractall(model_dir) archive.close() self.reader = MultiWozReader() self.m = DAMD(self.reader) if cfg.cuda and torch.cuda.is_available(): self.m = self.m.cuda() #cfg.cuda_device[0] if cfg.limit_bspn_vocab: self.reader.bspn_masks_tensor = {} for key, values in self.reader.bspn_masks.items(): v_ = cuda_(torch.Tensor(values).long()) self.reader.bspn_masks_tensor[key] = v_ if cfg.limit_aspn_vocab: self.reader.aspn_masks_tensor = {} for key, values in self.reader.aspn_masks.items(): v_ = cuda_(torch.Tensor(values).long()) self.reader.aspn_masks_tensor[key] = v_ cfg.model_path = os.path.join(model_path, 'model.pkl') self.load_model(cfg.model_path) self.m.eval() self.init_session()
def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, model_file='https://convlab.blob.core.windows.net/convlab-2/mle_policy_crosswoz.zip'): root_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f: cfg = json.load(f) voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json') voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json') self.vector = CrossWozVector(sys_da_voc_json=voc_file, usr_da_voc_json=voc_opp_file) self.policy = MultiDiscretePolicy(self.vector.state_dim, cfg['h_dim'], self.vector.sys_da_dim).to(device=DEVICE) if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MLE Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.load_from_pretrained(archive_file, model_file, cfg['load'])
def load_weights(self): DEFAULT_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models") DEFAULT_ARCHIVE_FILE = os.path.join(DEFAULT_DIRECTORY, "sumbt_multiwoz.zip") archive_file = DEFAULT_ARCHIVE_FILE model_file = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/sumbt_multiwoz.zip' if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for SC-LSTM is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'resource')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) model_ckpt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pytorch_model.bin') # model_ckpt = os.path.join(OUTPUT_DIR, "pytorch_model.bin") model = self.belief_tracker # in the case that slot and values are different between the training and evaluation ptr_model = torch.load(model_ckpt) print('loading pretrained weights') if N_GPU == 1: state = model.state_dict() state.update(ptr_model) model.load_state_dict(state) else: # print("Evaluate using only one device!") model.module.load_state_dict(ptr_model) model.to(DEVICE)
def __init__(self, mode): """ SVM NLU initialization. Args: mode (str): can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on. Example: nlu = SVMNLU(mode='usr') """ assert mode == 'usr' or mode == 'sys' or mode == 'all' model_file = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/svm_multiwoz_{}.zip'.format(mode) config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/multiwoz_{}.cfg'.format(mode)) self.config = configparser.ConfigParser() self.config.read(config_file) self.c = Classifier.classifier(self.config) model_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), self.config.get("train", "output")) model_dir = os.path.dirname(model_path) if not os.path.exists(model_path): if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) archive.close() self.c.load(model_path)
def __init__(self, archive_file, model_file=None, use_cuda=False): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for DA-predictor is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'checkpoints')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) load_dir = os.path.join(model_dir, "checkpoints/predictor/save_step_15120") self.db = Database() if not os.path.exists(load_dir): archive = zipfile.ZipFile(f'{load_dir}.zip', 'r') archive.extractall(os.path.dirname(load_dir)) self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True) self.max_seq_length = 256 self.domain = 'restaurant' self.model = BertForSequenceClassification.from_pretrained( load_dir, cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(-1)), num_labels=44) self.device = 'cuda' if use_cuda else 'cpu' self.model.to(self.device)
def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, model_file=None, context_size=3): """ Constructor for NLU class. """ self.context_size = context_size check_for_gpu(cuda_device) if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MILU is specified!") archive_file = cached_path(model_file) archive = load_archive(archive_file, cuda_device=cuda_device) self.tokenizer = SpacyWordSplitter(language="en_core_web_sm") _special_case = [{ORTH: u"id", LEMMA: u"id"}] self.tokenizer.spacy.tokenizer.add_special_case(u"id", _special_case) dataset_reader_params = archive.config["dataset_reader"] self.dataset_reader = DatasetReader.from_params(dataset_reader_params) self.model = archive.model self.model.eval()
def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, use_cuda=False, is_user=False, model_file='https://convlab.blob.core.windows.net/convlab-2/nlg_sclstm_camrest.zip'): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for SC-LSTM is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'resource')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.USE_CUDA = use_cuda self.args, self.config = parse(is_user) self.dataset = SimpleDatasetCam(self.config) # get model hyper-parameters hidden_size = self.config.getint('MODEL', 'hidden_size') # get feat size d_size = self.dataset.da_size + self.dataset.sv_size # len of 1-hot feat vocab_size = len(self.dataset.word2index) self.model = LMDeep('sclstm', vocab_size, vocab_size, hidden_size, d_size, n_layer=self.args['n_layer'], use_cuda=use_cuda) model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.args['model_path']) # print(model_path) assert os.path.isfile(model_path) self.model.load_state_dict(torch.load(model_path)) self.model.eval() if use_cuda: self.model.cuda()
def load_from_pretrained(self, archive_file, model_file, filename): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for PPO Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_ppo.pol.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) policy_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_ppo.pol.mdl') if os.path.exists(policy_mdl): self.policy.load_state_dict( torch.load(policy_mdl, map_location=DEVICE)) logging.info( '<<dialog policy>> loaded checkpoint from file: {}'.format( policy_mdl)) value_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_ppo.val.mdl') if os.path.exists(value_mdl): self.value.load_state_dict( torch.load(value_mdl, map_location=DEVICE)) logging.info( '<<dialog policy>> loaded checkpoint from file: {}'.format( value_mdl))
def __init__(self, model_file=DEFAULT_MODEL_URL, name='Sequicity'): """ Sequicity initialization Args: model_file (str): trained model path or url. default="https://convlab.blob.core.windows.net/convlab-2/sequicity_multiwoz.zip" Example: sequicity = Sequicity() """ super(Sequicity, self).__init__(name=name) nltk.download('punkt') config_file = DEFAULT_CONFIG_FILE c = json.load(open(config_file)) cfg.init_handler(c['tsdf_init']) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/data')): print('down load data from', DEFAULT_ARCHIVE_FILE_URL) archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'multiwoz/')) archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/')) archive.close() model_path = os.path.join(DEFAULT_DIRECTORY, c['tsdf_init']['model_path']) if not os.path.exists(model_path): model_dir = os.path.dirname(model_path) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') print('down load data from', model_file) archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', model_dir) archive.extractall(model_dir) archive.close() torch.manual_seed(cfg.seed) torch.cuda.manual_seed(cfg.seed) random.seed(cfg.seed) np.random.seed(cfg.seed) self.m = Model('multiwoz') self.m.count_params() self.m.load_model() self.init_session()
def __init__( self, mode='all', config_file='multiwoz_all_context.json', model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_all_context.zip' ): assert mode == 'usr' or mode == 'sys' or mode == 'all' self.mode = mode config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) # print(config['DEVICE']) # DEVICE = config['DEVICE'] DEVICE = 'cpu' if not torch.cuda.is_available() else 'cuda:0' root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load( open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader( intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) best_model_path = os.path.join(output_dir, 'pytorch_model.bin') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim).cuda() model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), map_location=lambda storage, loc: storage.cuda( torch.cuda.current_device()))) # model.to(DEVICE) model.eval() self.model = model self.use_context = config['model']['context'] self.dataloader = dataloader self.nlp = spacy.load('en_core_web_sm') print("BERTNLU loaded")
def __init__(self, model_file=DEFAULT_MODEL_URL): """ Sequicity initialization Args: model_file (str): trained model path or url. default="https://tatk-data.s3-ap-northeast-1.amazonaws.com/sequicity_camrest.zip" Example: sequicity = Sequicity() """ config_file = DEFAULT_CONFIG_FILE c = json.load(open(config_file)) cfg.init_handler(c['tsdf_init']) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY, 'camrest/data')): print('down load data from', DEFAULT_ARCHIVE_FILE_URL) archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'camrest/')) archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'camrest/')) archive.close() model_path = os.path.join(DEFAULT_DIRECTORY, c['tsdf_init']['model_path']) if not os.path.exists(model_path): model_dir = os.path.dirname(model_path) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Load from model_file param') print('down load data from', model_file) archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') print('unzip to', model_dir) archive.extractall(model_dir) archive.close() torch.manual_seed(cfg.seed) torch.cuda.manual_seed(cfg.seed) random.seed(cfg.seed) np.random.seed(cfg.seed) self.m = Model('camrest') self.m.count_params() self.m.load_model() self.init_session()
def auto_download(self): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.config_path, 'model/rnn_model_state_dict.th')) and \ os.path.exists(os.path.join(self.config_path, 'selection_model_state_dict.th')): return models_dir = os.path.join(self.config_path, 'models') cached_path(self.file_url, models_dir) files = os.listdir(models_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print('allennlp download file error: RnnRollout Deal_or_Not data download failed.') raise e zip_file_path = os.path.join(models_dir, target_file + '.zip') shutil.copyfile(os.path.join(models_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(models_dir)
def download_model(self): if not os.path.isdir(DOWNLOAD_DIRECTORY): os.mkdir(DOWNLOAD_DIRECTORY) # model_file = os.path.join(DOWNLOAD_DIRECTORY, 'pytorch_model.zip') # if not os.path.isfile(model_file): model_file = 'https://convlab.blob.core.windows.net/convlab-2/sumbt.tar.gz' import tarfile if not os.path.isfile(os.path.join(DOWNLOAD_DIRECTORY, 'pytorch_model.bin')): archive_file = cached_path(model_file) # archive = zipfile.ZipFile(archive_file, 'r') t = tarfile.open(archive_file) t.extractall(path=DOWNLOAD_DIRECTORY)
def __init__( self, mode='all', config_file='crosswoz_all_context.json', model_file='https://convlab.blob.core.windows.net/convlab-2/bert_crosswoz_all_context.zip' ): assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode, config['model']['pretrained_weights']) intent_vocab = json.load( open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader( intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) best_model_path = os.path.join(output_dir, 'pytorch_model.bin') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
def auto_download(self): """Automatically download the pretrained model and necessary data.""" if os.path.exists(os.path.join(self.data_dir, 'models')) and \ os.path.exists(os.path.join(self.data_dir, 'data')) and \ os.path.exists(os.path.join(self.data_dir, 'word-vectors')): return cached_path(self.file_url, self.data_dir) files = os.listdir(self.data_dir) target_file = '' for name in files: if name.endswith('.json'): target_file = name[:-5] try: assert target_file in files except Exception as e: print( 'allennlp download file error: MDBT Multiwoz data download failed.' ) raise e zip_file_path = os.path.join(self.data_dir, target_file + '.zip') shutil.copyfile(os.path.join(self.data_dir, target_file), zip_file_path) with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(self.data_dir)
def load(self, archive_file, model_file, filename): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for VHUS Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_simulator.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) user_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_simulator.mdl') if os.path.exists(user_mdl): DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.user.load_state_dict(torch.load(user_mdl, map_location=DEVICE)) print('<<user simulator>> loaded checkpoint from file: {}'.format(user_mdl))
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, use_cuda=False, is_user=False, model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/nlg_sclstm_multiwoz.zip' ): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for SC-LSTM is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'resource')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.USE_CUDA = use_cuda self.args, self.config = parse(is_user) self.dataset = SimpleDatasetWoz(self.config) # get model hyper-parameters hidden_size = self.config.getint('MODEL', 'hidden_size') # get feat size d_size = self.dataset.do_size + self.dataset.da_size + self.dataset.sv_size # len of 1-hot feat vocab_size = len(self.dataset.word2index) self.model = LMDeep('sclstm', vocab_size, vocab_size, hidden_size, d_size, n_layer=self.args['n_layer'], use_cuda=use_cuda) model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.args['model_path']) # print(model_path) assert os.path.isfile(model_path) self.model.load_state_dict(torch.load(model_path)) self.model.eval() for name, param in self.model.named_parameters(): print(name, param.shape, param.device, param.requires_grad) if use_cuda: self.model.cuda()
def auto_download(): model_path = os.path.join(os.path.dirname(__file__), 'model') data_path = os.path.join(os.path.dirname(__file__), 'data') db_path = os.path.join(os.path.dirname(__file__), 'db') root_path = os.path.dirname(__file__) urls = {model_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_model.zip', data_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_data.zip', db_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_db.zip'} for path in [model_path, data_path, db_path]: if not os.path.exists(path): file_url = urls[path] print('Downloading from: ', file_url) archive_file = cached_path(file_url) print('Extracting...') archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_path)
def __init__(self, archive_file, model_file=None, use_cuda=False): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for HDSA is specified!") archive_file = cached_path(model_file) model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.exists(os.path.join(model_dir, 'checkpoints')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) with open(os.path.join(model_dir, "data/vocab.json"), 'r') as f: vocabulary = json.load(f) vocab, ivocab = vocabulary['vocab'], vocabulary['rev'] self.tokenizer = Tokenizer(vocab, ivocab, False) self.max_seq_length = 50 self.history = [] self.kbs = {} self.decoder = TableSemanticDecoder( vocab_size=self.tokenizer.vocab_len, d_word_vec=128, n_layers=3, d_model=128, n_head=4, dropout=0.2) self.device = 'cuda' if use_cuda else 'cpu' self.decoder.to(self.device) checkpoint_file = os.path.join( model_dir, "checkpoints/generator/BERT_dim128_w_domain_exp") if use_cuda: self.decoder.load_state_dict(torch.load(checkpoint_file)) else: self.decoder.load_state_dict(torch.load(checkpoint_file, 'cpu')) with open(os.path.join(model_dir, 'data/svdic.pkl'), 'rb') as f: self.dic = pickle.load(f) with open(os.path.join(model_dir, 'data/mapping.pair')) as f: self.replacements = [] for line in f: tok_from, tok_to = line.replace('\n', '').split('\t') self.replacements.append( (' ' + tok_from + ' ', ' ' + tok_to + ' '))
def load(self, archive_file, model_file, filename): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MLE Policy is specified!") archive_file = cached_path(model_file) model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save') if not os.path.exists(model_dir): os.mkdir(model_dir) if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')): archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) policy_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_mle.pol.mdl') if os.path.exists(policy_mdl): self.policy.load_state_dict(torch.load(policy_mdl)) print('<<dialog policy>> loaded checkpoint from file: {}'.format( policy_mdl))
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, model_file="https://convlab.blob.core.windows.net/convlab-2/new_milu(20200922)_multiwoz_all_context.tar.gz", context_size=3): """ Constructor for NLU class. """ self.context_size = context_size cuda_device = 0 if torch.cuda.is_available() else DEFAULT_CUDA_DEVICE check_for_gpu(cuda_device) if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MILU is specified!") archive_file = cached_path(model_file) archive = load_archive(archive_file, cuda_device=cuda_device) self.tokenizer = SpacyWordSplitter(language="en_core_web_sm") _special_case = [{ORTH: u"id", LEMMA: u"id"}] self.tokenizer.spacy.tokenizer.add_special_case(u"id", _special_case) with open( os.path.join(get_root_path(), 'data/multiwoz/db/postcode.json'), 'r') as f: token_list = json.load(f) for token in token_list: token = token.strip() self.tokenizer.spacy.tokenizer.add_special_case( token, [{ ORTH: token, LEMMA: token, POS: u'NOUN' }]) dataset_reader_params = archive.config["dataset_reader"] self.dataset_reader = DatasetReader.from_params(dataset_reader_params) self.model = archive.model self.model.eval()
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, use_cuda=True, is_user=False, model_file='https://convlab.blob.core.windows.net/convlab-2/nlg-gpt-multiwoz.zip' ): model_dir = os.path.dirname(os.path.abspath(__file__)) if not os.path.isfile(archive_file): archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(model_dir) self.model_name_or_path = os.path.join(model_dir, 'multiwoz') self.length = 50 self.num_samples = 5 self.temperature = 1.0 self.repetition_penalty = 1.0 self.top_k = 50 self.top_p = 0.9 self.seed = 42 self.stop_token = '<|endoftext|>' self.device = torch.device( "cuda" if torch.cuda.is_available() and use_cuda else "cpu") set_seed(self.seed, torch.cuda.device_count()) model_class, tokenizer_class = GPT2LMHeadModel, GPT2Tokenizer self.tokenizer = tokenizer_class.from_pretrained( self.model_name_or_path) self.model = model_class.from_pretrained(self.model_name_or_path) self.model.to(self.device) self.model.eval() if self.length < 0 and self.model.config.max_position_embeddings > 0: self.length = self.model.config.max_position_embeddings elif 0 < self.model.config.max_position_embeddings < self.length: self.length = self.model.config.max_position_embeddings # No generation bigger than model size elif self.length < 0: self.length = self.MAX_LENGTH # avoid infinite loop
def auto_download(): model_path = os.path.join(os.path.dirname(__file__), 'model') data_path = os.path.join(os.path.dirname(__file__), 'data') db_path = os.path.join(os.path.dirname(__file__), 'db') root_path = os.path.dirname(__file__) urls = { model_path: 'https://convlab.blob.core.windows.net/convlab-2/mdrg_model.zip', data_path: 'https://convlab.blob.core.windows.net/convlab-2/mdrg_data.zip', db_path: 'https://convlab.blob.core.windows.net/convlab-2/mdrg_db.zip' } for path in [model_path, data_path, db_path]: if not os.path.exists(path): file_url = urls[path] print('Downloading from: ', file_url) archive_file = cached_path(file_url) print('Extracting...') archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_path)
def __init__( self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, model_file="https://convlab.blob.core.windows.net/convlab-2/larl.zip" ): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for LaRL is specified!") archive_file = cached_path(model_file) temp_path = os.path.dirname(os.path.abspath(__file__)) #print(temp_path) zip_ref = zipfile.ZipFile(archive_file, 'r') zip_ref.extractall(temp_path) zip_ref.close() self.prev_state = default_state() self.prev_active_domain = None domain_name = 'object_division' domain_info = domain.get_domain(domain_name) self.db = Database() data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data') train_data_path = os.path.join(data_path, 'train_dials.json') if not os.path.exists(train_data_path): zipped_file = os.path.join(data_path, 'norm-multi-woz.zip') archive = zipfile.ZipFile(zipped_file, 'r') archive.extractall(data_path) norm_multiwoz_path = data_path with open( os.path.join(norm_multiwoz_path, 'input_lang.index2word.json')) as f: self.input_lang_index2word = json.load(f) with open( os.path.join(norm_multiwoz_path, 'input_lang.word2index.json')) as f: self.input_lang_word2index = json.load(f) with open( os.path.join(norm_multiwoz_path, 'output_lang.index2word.json')) as f: self.output_lang_index2word = json.load(f) with open( os.path.join(norm_multiwoz_path, 'output_lang.word2index.json')) as f: self.output_lang_word2index = json.load(f) config = Pack( seed=10, train_path=train_data_path, max_vocab_size=1000, last_n_model=5, max_utt_len=50, max_dec_len=50, backward_size=2, batch_size=1, use_gpu=True, op='adam', init_lr=0.001, l2_norm=1e-05, momentum=0.0, grad_clip=5.0, dropout=0.5, max_epoch=100, embed_size=100, num_layers=1, utt_rnn_cell='gru', utt_cell_size=300, bi_utt_cell=True, enc_use_attn=True, dec_use_attn=True, dec_rnn_cell='lstm', dec_cell_size=300, dec_attn_mode='cat', y_size=10, k_size=20, beta=0.001, simple_posterior=True, contextual_posterior=True, use_mi=False, use_pr=True, use_diversity=False, # beam_size=20, fix_batch=True, fix_train_batch=False, avg_type='word', print_step=300, ckpt_step=1416, improve_threshold=0.996, patient_increase=2.0, save_model=True, early_stop=False, gen_type='greedy', preview_batch_num=None, k=domain_info.input_length(), init_range=0.1, pretrain_folder='2019-09-20-21-43-06-sl_cat', forward_only=False) config.use_gpu = config.use_gpu and torch.cuda.is_available() self.corpus = corpora_inference.NormMultiWozCorpus(config) self.model = SysPerfectBD2Cat(self.corpus, config) self.config = config if config.use_gpu: self.model.load_state_dict( torch.load(os.path.join(temp_path, 'larl_model/best-model'))) self.model.cuda() else: self.model.load_state_dict( torch.load(os.path.join(temp_path, 'larl_model/best-model'), map_location=lambda storage, loc: storage)) self.model.eval() self.dic = pickle.load( open(os.path.join(temp_path, 'larl_model/svdic.pkl'), 'rb'))
def _read(self, file_path): # if `file_path` is a URL, redirect to the cache file_path = cached_path(file_path) if file_path.endswith("zip"): archive = zipfile.ZipFile(file_path, "r") data_file = archive.open(os.path.basename(file_path)[:-4]) else: data_file = open(file_path, "r") logger.info("Reading instances from lines in file at: %s", file_path) dialogs = json.load(data_file) for dial_name in dialogs: dialog = dialogs[dial_name]["log"] context_tokens_list = [] for i, turn in enumerate(dialog): if self._agent and self._agent == "user" and i % 2 != 1: continue if self._agent and self._agent == "system" and i % 2 != 0: continue tokens = turn["text"].split() dialog_act = {} for dacts in turn["span_info"]: if dacts[0] not in dialog_act: dialog_act[dacts[0]] = [] dialog_act[dacts[0]].append( [dacts[1], " ".join(tokens[dacts[3]:dacts[4] + 1])]) spans = turn["span_info"] tags = [] for j in range(len(tokens)): for span in spans: if j == span[3]: tags.append("B-" + span[0] + "+" + span[1]) break if j > span[3] and j <= span[4]: tags.append("I-" + span[0] + "+" + span[1]) break else: tags.append("O") intents = [] for dacts in turn["dialog_act"]: for dact in turn["dialog_act"][dacts]: if dacts not in dialog_act or dact[0] not in [ sv[0] for sv in dialog_act[dacts] ]: if dact[1] in [ "none", "?", "yes", "no", "do nt care", "do n't care" ]: intents.append(dacts + "+" + dact[0] + "*" + dact[1]) for dacts in turn["dialog_act"]: for dact in turn["dialog_act"][dacts]: if dacts not in dialog_act: dialog_act[dacts] = turn["dialog_act"][dacts] break elif dact[0] not in [ sv[0] for sv in dialog_act[dacts] ]: dialog_act[dacts].append(dact) num_context = random.randint( 0, self._context_size ) if self._random_context_size else self._context_size if len(context_tokens_list) > 0 and num_context > 0: wrapped_context_tokens = [ Token(token) for context_tokens in context_tokens_list[-num_context:] for token in context_tokens ] else: wrapped_context_tokens = [Token("SENT_END")] wrapped_tokens = [Token(token) for token in tokens] context_tokens_list.append(tokens + ["SENT_END"]) yield self.text_to_instance(wrapped_context_tokens, wrapped_tokens, tags, intents, dialog_act)
def __init__( self, mode='all', config_file='multiwoz_all_context.json', model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_all_context.zip' ): assert mode == 'usr' or mode == 'sys' or mode == 'all' self.mode = mode config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) # print(config['DEVICE']) # DEVICE = config['DEVICE'] DEVICE = 'cpu' if not torch.cuda.is_available() else 'cuda:0' root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load( open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader( intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) best_model_path = os.path.join(output_dir, 'pytorch_model.bin') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.use_context = config['model']['context'] self.dataloader = dataloader try: self.nlp = spacy.load("en_core_web_sm") except Exception: print('download en_core_web_sm for spacy') from spacy.cli.download import download as spacy_download spacy_download("en_core_web_sm") spacy_model_module = __import__("en_core_web_sm") self.nlp = spacy_model_module.load() with open( os.path.join(get_root_path(), 'data/multiwoz/db/postcode.json'), 'r') as f: token_list = json.load(f) for token in token_list: token = token.strip() self.nlp.tokenizer.add_special_case(token, [{ ORTH: token, LEMMA: token, POS: u'NOUN' }]) print("BERTNLU loaded")
import os os.environ["CUDA_VISIBLE_DEVICES"] = "0" from .utils.config import * from .models.TRADE import * from convlab2.util.file_util import cached_path ''' python3 myTest.py -ds= -path= -bsz= ''' # download pretrained model cached_path(args['model_url'], args['model_path']) import zipfile with zipfile.ZipFile(args['model_path'] + 'trade_crosswoz_model.zip', 'r') as zip_ref: zip_ref.extractall(args['model_path']) directory = args['path'].split("/") HDD = directory[2].split('HDD')[1].split('BSZ')[0] decoder = directory[1].split('-')[0] BSZ = int(args['batch']) if args['batch'] else int( directory[2].split('BSZ')[1].split('DR')[0]) args["decoder"] = decoder args["HDD"] = HDD print("HDD", HDD, "decoder", decoder, "BSZ", BSZ) if args['dataset'] == 'multiwoz': from .utils.utils_multiWOZ_DST import * else: print("You need to provide the --dataset information")