def classify(X):
global classifier
model_name = 'TC_GENERAL_V131'
model_path = ModelFetcher.get_model_path(model_name)
if not classifier:
if not os.path.exists(model_path):
ModelFetcher.download(model_name)
classifier = TextClassifier.load(model_path)
sentence = Sentence(X)
classifier.predict(sentence)
labels = sentence.labels
return labels
def classify(X):
global classifier
if not classifier:
if os.path.exists(model_path):
classifier = TextClassifier.load(model_path)
else:
logger.error(
f"Could not load model at {model_path}.\n"
f"Download model with \"underthesea download {UTSModel.tc_general.value}\".")
sys.exit(1)
sentence = Sentence(X)
classifier.predict(sentence)
labels = sentence.labels
return labels
def sentiment(X):
global classifier
model_name = 'SA_BANK_V131'
model_path = ModelFetcher.get_model_path(model_name)
if not classifier:
if not os.path.exists(model_path):
ModelFetcher.download(model_name)
classifier = TextClassifier.load(model_path)
sentence = Sentence(X)
classifier.predict(sentence)
labels = sentence.labels
if not labels:
return None
labels = [label.value for label in labels]
return labels
def sentiment(text):
global classifier
if not classifier:
if os.path.exists(model_path):
classifier = TextClassifier.load(model_path)
else:
logger.error(
f"Could not load model at {model_path}.\n"
f"Download model with \"underthesea download {UTSModel.sa_bank.value}\"."
)
sys.exit(1)
sentence = Sentence(text)
classifier.predict(sentence)
labels = sentence.labels
if labels is None:
return None
return [label.value for label in labels]
def sentiment(text):
global classifier
if not classifier:
if os.path.exists(model_path):
classifier = TextClassifier.load(model_path)
else:
logger.error(
f"Could not load model at {model_path}.\n"
f"Download model with \"underthesea download {UTSModel.sa_general.value}\"."
)
sys.exit(1)
sentence = Sentence(text)
classifier.predict(sentence)
label = sentence.labels[0]
if label == "1":
label = "negative"
if label == "0":
label = "positive"
return label
def sentiment(X):
global classifier
model_name = 'SA_GENERAL_V131'
model_path = ModelFetcher.get_model_path(model_name)
if not classifier:
if not os.path.exists(model_path):
ModelFetcher.download(model_name)
classifier = TextClassifier.load(model_path)
sentence = Sentence(X)
classifier.predict(sentence)
labels = sentence.labels
try:
label_map = {'POS': 'positive', 'NEG': 'negative'}
label = label_map[labels[0]]
return label
except Exception:
return None
model_folder = "tmp/sentiment_svm_ubs"
shutil.rmtree(model_folder, ignore_errors=True)
os.makedirs(model_folder)
start = time.time()
print(">>> Train UBS model")
data_folder = Path(join(DATASETS_FOLDER, "SE_Vietnamese-UBS-1"))
corpus: CategorizedCorpus = DataFetcher.load_classification_corpus(data_folder)
print("\n\n>>> Sample sentences")
for s in corpus.train[:10]:
print(s)
pipeline = Pipeline(
steps=[('features', TfidfVectorizer(ngram_range=(
1, 2), max_df=0.5)), ('estimator', OneVsRestClassifier(LinearSVC()))])
print("\n\n>>> Start training")
classifier = TextClassifier(estimator=TEXT_CLASSIFIER_ESTIMATOR.PIPELINE,
pipeline=pipeline,
multilabel=True)
model_trainer = ClassifierTrainer(classifier, corpus)
def micro_f1_score(y_true, y_pred):
return f1_score(y_true, y_pred, average='micro')
model_trainer.train(model_folder, scoring=micro_f1_score)
print(f"\n\n>>> Finish training in {round(time.time() - start, 2)} seconds")
print(f"Your model is saved in {model_folder}")
from underthesea.corpus.data import Sentence
from underthesea.models.text_classifier import TextClassifier
model_folder = "tmp/classification_svm_vntc"
print(f"Load model from {model_folder}")
classifer = TextClassifier.load(model_folder)
print(f"Model is loaded.")
def predict(text):
print(f"\nText: {text}")
sentence = Sentence(text)
classifer.predict(sentence)
labels = sentence.labels
print(f"Labels: {labels}")
predict('Huawei có thể không cần Google, nhưng sẽ ra sao nếu thiếu ARM ?')
predict(
'Trưởng phòng GD&ĐT xin lỗi vụ học sinh nhận khen thưởng là tờ giấy A4')