class HA(BaseTextModel):
"""Wrapper of b4msa.textmodel.TextModel and LinearSVC"""
def __init__(self, **kwargs):
self._tm = TextModel(**kwargs)
self._cl = LinearSVC()
def fit(self, X, y):
self._tm.fit(X)
self._cl.fit(self._tm.transform(X), y)
return self
def tonp(self, X):
return X
def transform(self, X):
res = self._cl.decision_function(self._tm.transform(X))
if res.ndim == 1:
return np.atleast_2d(res).T
return res
@classmethod
def create_space(cls, fname, output, **kwargs):
"""Create the model from a file of json
:param fname: Path to the file containing the json
:type fname: str
:param output: Path to store the model
:type output: str
:param kwargs: Keywords pass to TextModel
"""
X = [x for x in tweet_iterator(fname)]
m = cls(**kwargs)
m.fit(X, [x['klass'] for x in X])
save_model(m, output)
D = [(x['text'], x['klass']) for x in tweet_iterator(TWEETS)]
y = [y for _, y in D]
le = LabelEncoderWrapper().fit(y)
y = le.transform(y)
tm = TextModel(token_list=[-1],
weighting='tf').fit([x for x, _ in D])
folds = StratifiedKFold(shuffle=True, random_state=0)
hy = np.empty(len(D))
for tr, val in folds.split(D, y):
_ = [D[x][0] for x in tr]
X = tm.transform(_)
m = LogisticRegression(multi_class='multinomial').fit(X, y[tr])
# m = LinearSVC().fit(X, y[tr])
_ = [D[x][0] for x in val]
hy[val] = m.predict(tm.transform(_))
ci = bootstrap_confidence_interval(y, hy)
ci
(0.2839760475399691, 0.30881116416736665)
tm = TextModel(token_list=[-1]).fit([x for x, _ in D])
hy = np.empty(len(D))
for tr, val in folds.split(D, y):
_ = [D[x][0] for x in tr]
X = tm.transform(_)
m = LogisticRegression(multi_class='multinomial').fit(X, y[tr])
from sklearn.svm import LinearSVC
from sklearn.preprocessing import LabelEncoder
import numpy as np
from microtc.utils import tweet_iterator
from b4msa.textmodel import TextModel
from sklearn.metrics import recall_score
from sklearn.model_selection import KFold
from EvoMSA.utils import bootstrap_confidence_interval
D = list(tweet_iterator("semeval2017_En_train.json"))
tm = TextModel(lang="english").fit(D)
# ,
# token_list=[-1]).fit(D)
le = LabelEncoder().fit([x['klass'] for x in D])
X = tm.transform(D)
y = le.transform([x['klass'] for x in D])
m = LinearSVC().fit(X, y)
recall_score(y, m.predict(X), average=None)
kf = KFold()
hy = np.empty_like(y)
for train, test in kf.split(X):
m = LinearSVC().fit(X[train], y[train])
hy[test] = m.predict(X[test])
bootstrap_confidence_interval(
y, hy, metric=lambda y, hy: recall_score(y, hy, average=None)[0])
