def __init__(self, model_dir: str, params: dict, verbose: int=0): self.config = Config(model_dir, params, verbose=verbose) self.verbose=verbose if not 'memory' in self.config['params']: self.config['params']['memory'] = {} if not 'max_size_mb' in self.config['params']['memory']: self.config['params']['memory']['max_size_mb'] = 2 if not 'max_size_train_samples' in self.config['params']['memory']: self.config['params']['memory']['max_size_train_samples'] = 10000 if not 'field_target_name' in self.config['params']: self.config['params']['field_target_name'] = 'target'
def main(args): # load configs dconf_path = 'config/data.json' mconf_path = 'config/word2vec.json' dconf = Config(dconf_path) mconf = Config(mconf_path) # load w2v model and train if mconf.model == 'cbow': w2v = CbowModel(dconf, mconf, args.mode) else: w2v = SkipGramModel(dconf, mconf, args.mode) if args.mode != 'test': w2v.train() w2v.save(dconf.saved_file) # test w2v word = 'hospital' print(w2v.nearest(word)) print(w2v.similarity(word, 'attacks').item()) print(w2v.similarity(word, word).item())
def main(parse): # load configs dconf_path = 'config/data.json' mconf_path = 'config/word2vec.json' dconf = Config(dconf_path) mconf = Config(mconf_path) # load w2v model and train if mconf.model == 'cbow': w2v = CbowModel(dconf, mconf) else: w2v = SkipGramModel(dconf, mconf) w2v.load('trained.pth') # test w2v word = 'hospital' print(w2v.nearest(word)) print(w2v.similarity(word, 'attacks').item()) print(w2v.similarity(word, word).item())
import pickle from lib.util import Config from lib.kor2eng import LangTranslator from lib.util import load_data from lib.data_preprocess import Vocab, preprocessor from lib.model.seq2seq import BiLSTMSeq2Seq from transformers.lib.model.transformer import Transformer file_dir = os.path.dirname(__file__) sys.path.append(file_dir) # load configs dconf_path = 'config/data.json' mconf_path = 'config/lm.json' dconf = Config(dconf_path) mconf = Config(mconf_path) device = torch.device("cuda") if torch.cuda.is_available() else torch.device( "cpu") print('Using device:', device) # try: # with open('preprocessed_data.pickle', 'rb') as f: # saved_obj = pickle.load(f) # ko_corpus, ko_vocab, en_corpus, en_vocab = saved_obj # except: # # # load & preprocess corpus # ko_corpus = preprocessor(load_data(dconf.train_ko_path), lang='ko') # en_corpus = preprocessor(load_data(dconf.train_en_path), lang='en')
def __init__(self, model_dir: str): os.makedirs(model_dir, exist_ok=True) self.config = Config(model_dir)