def __init__(self, model_dir: str, params: dict, verbose: int=0):
self.config = Config(model_dir, params, verbose=verbose)
self.verbose=verbose
if not 'memory' in self.config['params']:
self.config['params']['memory'] = {}
if not 'max_size_mb' in self.config['params']['memory']:
self.config['params']['memory']['max_size_mb'] = 2
if not 'max_size_train_samples' in self.config['params']['memory']:
self.config['params']['memory']['max_size_train_samples'] = 10000
if not 'field_target_name' in self.config['params']:
self.config['params']['field_target_name'] = 'target'
def main(args):
# load configs
dconf_path = 'config/data.json'
mconf_path = 'config/word2vec.json'
dconf = Config(dconf_path)
mconf = Config(mconf_path)
# load w2v model and train
if mconf.model == 'cbow':
w2v = CbowModel(dconf, mconf, args.mode)
else:
w2v = SkipGramModel(dconf, mconf, args.mode)
if args.mode != 'test':
w2v.train()
w2v.save(dconf.saved_file)
# test w2v
word = 'hospital'
print(w2v.nearest(word))
print(w2v.similarity(word, 'attacks').item())
print(w2v.similarity(word, word).item())
def main(parse):
# load configs
dconf_path = 'config/data.json'
mconf_path = 'config/word2vec.json'
dconf = Config(dconf_path)
mconf = Config(mconf_path)
# load w2v model and train
if mconf.model == 'cbow':
w2v = CbowModel(dconf, mconf)
else:
w2v = SkipGramModel(dconf, mconf)
w2v.load('trained.pth')
# test w2v
word = 'hospital'
print(w2v.nearest(word))
print(w2v.similarity(word, 'attacks').item())
print(w2v.similarity(word, word).item())
import pickle
from lib.util import Config
from lib.kor2eng import LangTranslator
from lib.util import load_data
from lib.data_preprocess import Vocab, preprocessor
from lib.model.seq2seq import BiLSTMSeq2Seq
from transformers.lib.model.transformer import Transformer
file_dir = os.path.dirname(__file__)
sys.path.append(file_dir)
# load configs
dconf_path = 'config/data.json'
mconf_path = 'config/lm.json'
dconf = Config(dconf_path)
mconf = Config(mconf_path)
device = torch.device("cuda") if torch.cuda.is_available() else torch.device(
"cpu")
print('Using device:', device)
# try:
# with open('preprocessed_data.pickle', 'rb') as f:
# saved_obj = pickle.load(f)
# ko_corpus, ko_vocab, en_corpus, en_vocab = saved_obj
# except:
#
# # load & preprocess corpus
# ko_corpus = preprocessor(load_data(dconf.train_ko_path), lang='ko')
# en_corpus = preprocessor(load_data(dconf.train_en_path), lang='en')
def __init__(self, model_dir: str):
os.makedirs(model_dir, exist_ok=True)
self.config = Config(model_dir)