def __init__(self, embed_model_path=default_embed_model_path, cls_model_path=default_cls_model_path): if not os.path.isfile(embed_model_path): print('embed_model doesn\'t exist.\ncheck the model path') if not os.path.isfile(cls_model_path): print('classifier_model doesn\'t exist.\ncheck the model path') # 전처리 패키지 초기화 print('Init preprocessing') initialize(KMR='2.1.4') #LATEST--> 2.1.4로 변경 self.tagger = Tagger(API.KMR) # 임베딩 모델 로딩 print('Loading Embedding model') self.embedding_model = Doc2Vec.load(embed_model_path) # Classifier 모델 로딩 print('Loading Classifier model') with open(cls_model_path, 'rb') as fp: self.clf_model = pickle.load(fp)
def init_koala_nlp(): initialize(HNN="LATEST")
from koalanlp.Util import initialize, finalize from koalanlp.proc import Parser from koalanlp import API initialize(KKMA='LATEST') #: HNN=2.0.4, ETRI=2.0.4 parser = Parser(API.KKMA) while True: text = input("분석할 문장을 입력하세요>> ").strip() if len(text) == 0: break sentences = parser(text) for sent in sentences: print("===== Sentence =====") print(sent.singleLineString()) print("# Dependency Parse result") dependencies = sent.getDependencies() if len(dependencies) > 0: for edge in dependencies: print("[%s]는 [%s]의 %s-%s" % (edge.getDependent().getSurface(), edge.getGovernor().getSurface() if edge.getGovernor() is not None else "ROOT", str( edge.getType()), str(edge.getDepType()))) else: print("(Unexpected) NULL!")
import torch import torch.nn.functional as F from gluonnlp.data import SentencepieceTokenizer from kogpt2.utils import get_tokenizer from kogpt2.utils import download, tokenizer from kogpt2.model.torch_gpt2 import GPT2LMHeadModel from kogpt2.configuration_gpt2 import GPT2Config import gluonnlp from koalanlp.Util import initialize, finalize from koalanlp.proc import Tagger from koalanlp import API from collections import Counter initialize(EUNJEON="LATEST") tagger = Tagger(API.EUNJEON) pytorch_kogpt2 = { "url": "./checkpoint/pytorch_kogpt2_676e9bcfa7.params", "fname": "pytorch_kogpt2_676e9bcfa7.params", "chksum": "676e9bcfa7", } kogpt2_config = { "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "n_ctx": 1024, "n_embd": 768, "n_head": 12, "n_layer": 12, "n_positions": 1024, "vocab_size": 50000,