def classify(X): global classifier model_name = 'TC_GENERAL_V131' model_path = ModelFetcher.get_model_path(model_name) if not classifier: if not os.path.exists(model_path): ModelFetcher.download(model_name) classifier = TextClassifier.load(model_path) sentence = Sentence(X) classifier.predict(sentence) labels = sentence.labels return labels
def classify(X): global classifier if not classifier: if os.path.exists(model_path): classifier = TextClassifier.load(model_path) else: logger.error( f"Could not load model at {model_path}.\n" f"Download model with \"underthesea download {UTSModel.tc_general.value}\".") sys.exit(1) sentence = Sentence(X) classifier.predict(sentence) labels = sentence.labels return labels
def sentiment(X): global classifier model_name = 'SA_BANK_V131' model_path = ModelFetcher.get_model_path(model_name) if not classifier: if not os.path.exists(model_path): ModelFetcher.download(model_name) classifier = TextClassifier.load(model_path) sentence = Sentence(X) classifier.predict(sentence) labels = sentence.labels if not labels: return None labels = [label.value for label in labels] return labels
def sentiment(text): global classifier if not classifier: if os.path.exists(model_path): classifier = TextClassifier.load(model_path) else: logger.error( f"Could not load model at {model_path}.\n" f"Download model with \"underthesea download {UTSModel.sa_bank.value}\"." ) sys.exit(1) sentence = Sentence(text) classifier.predict(sentence) labels = sentence.labels if labels is None: return None return [label.value for label in labels]
def sentiment(text): global classifier if not classifier: if os.path.exists(model_path): classifier = TextClassifier.load(model_path) else: logger.error( f"Could not load model at {model_path}.\n" f"Download model with \"underthesea download {UTSModel.sa_general.value}\"." ) sys.exit(1) sentence = Sentence(text) classifier.predict(sentence) label = sentence.labels[0] if label == "1": label = "negative" if label == "0": label = "positive" return label
def sentiment(X): global classifier model_name = 'SA_GENERAL_V131' model_path = ModelFetcher.get_model_path(model_name) if not classifier: if not os.path.exists(model_path): ModelFetcher.download(model_name) classifier = TextClassifier.load(model_path) sentence = Sentence(X) classifier.predict(sentence) labels = sentence.labels try: label_map = {'POS': 'positive', 'NEG': 'negative'} label = label_map[labels[0]] return label except Exception: return None
model_folder = "tmp/sentiment_svm_ubs" shutil.rmtree(model_folder, ignore_errors=True) os.makedirs(model_folder) start = time.time() print(">>> Train UBS model") data_folder = Path(join(DATASETS_FOLDER, "SE_Vietnamese-UBS-1")) corpus: CategorizedCorpus = DataFetcher.load_classification_corpus(data_folder) print("\n\n>>> Sample sentences") for s in corpus.train[:10]: print(s) pipeline = Pipeline( steps=[('features', TfidfVectorizer(ngram_range=( 1, 2), max_df=0.5)), ('estimator', OneVsRestClassifier(LinearSVC()))]) print("\n\n>>> Start training") classifier = TextClassifier(estimator=TEXT_CLASSIFIER_ESTIMATOR.PIPELINE, pipeline=pipeline, multilabel=True) model_trainer = ClassifierTrainer(classifier, corpus) def micro_f1_score(y_true, y_pred): return f1_score(y_true, y_pred, average='micro') model_trainer.train(model_folder, scoring=micro_f1_score) print(f"\n\n>>> Finish training in {round(time.time() - start, 2)} seconds") print(f"Your model is saved in {model_folder}")
from underthesea.corpus.data import Sentence from underthesea.models.text_classifier import TextClassifier model_folder = "tmp/classification_svm_vntc" print(f"Load model from {model_folder}") classifer = TextClassifier.load(model_folder) print(f"Model is loaded.") def predict(text): print(f"\nText: {text}") sentence = Sentence(text) classifer.predict(sentence) labels = sentence.labels print(f"Labels: {labels}") predict('Huawei có thể không cần Google, nhưng sẽ ra sao nếu thiếu ARM ?') predict( 'Trưởng phòng GD&ĐT xin lỗi vụ học sinh nhận khen thưởng là tờ giấy A4')