def test_lightgbmclassifier(self):
np.random.seed(0)
train_file = get_dataset('wiki_detox_train').as_filepath()
(train,
label) = get_X_y(train_file,
label_column='Sentiment',
sep='\t',
encoding="utf-8")
X_train, X_test, y_train, y_test = train_test_split(
train['SentimentText'], label)
# map text reviews to vector space
texttransform = NGramFeaturizer(
word_feature_extractor=n_gram(),
vector_normalizer='None') << 'SentimentText'
X_train = texttransform.fit_transform(X_train, max_slots=5000)
X_test = texttransform.transform(X_test, max_slots=5000)
mymodel = LightGbmClassifier().fit(X_train, y_train, verbose=0)
scores = mymodel.predict(X_test)
accuracy = np.mean(y_test.values.ravel() == scores.values)
assert_greater(
accuracy,
0.58,
"accuracy should be greater than %s" %
0.58)
def test_naivebayesclassifier(self):
np.random.seed(0)
train_file = get_dataset("wiki_detox_train").as_filepath()
(train, label) = get_X_y(train_file, label_column='Sentiment',
sep='\t')
X_train, X_test, y_train, y_test = train_test_split(
train['SentimentText'], label)
# map text reviews to vector space
texttransform = NGramFeaturizer(
word_feature_extractor=n_gram(),
vector_normalizer='None') << 'SentimentText'
X_train = texttransform.fit_transform(X_train)
X_test = texttransform.transform(X_test)
mymodel = NaiveBayesClassifier()
mymodel.fit(X_train, y_train)
scores = mymodel.predict(X_test)
accuracy = np.mean(y_test == [i for i in scores])[0]
assert_greater(
accuracy,
0.5,
"accuracy should be greater than %s" %
0.5)
def train_data_type_single(fit_X_type="dataframe",
fit_Y_type=None,
predict_X_type=None):
data = [
"This is sentence 1", "Talk about second", "Thrid one",
"Final example."
]
model = NGramFeaturizer()
data_with_new_type = transform_data(data, fit_X_type)
model.fit(data_with_new_type)
test_data_with_new_type = transform_data(data, predict_X_type)
return model.transform(test_data_with_new_type)
def test_ngramfeaturizer(self):
train_reviews = pandas.DataFrame(data=dict(
review=[
"This is great", "I hate it", "Love it", "Do not like it",
"Really like it", "I hate it", "I like it a lot",
"I kind of hate it", "I do like it", "I really hate it",
"It is very good", "I hate it a bunch", "I love it a bunch",
"I hate it", "I like it very much", "I hate it very much.",
"I really do love it", "I really do hate it", "Love it!",
"Hate it!", "I love it", "I hate it", "I love it", "I hate it",
"I love it"
],
like=[
True, False, True, False, True, False, True, False, True,
False, True, False, True, False, True, False, True, False,
True, False, True, False, True, False, True
]))
test_reviews = pandas.DataFrame(data=dict(review=[
"This is great", "I hate it", "Love it", "Really like it",
"I hate it", "I like it a lot", "I love it", "I do like it",
"I really hate it", "I love it"
]))
y = train_reviews['like']
X = train_reviews.loc[:, train_reviews.columns != 'like']
textt = NGramFeaturizer(word_feature_extractor=n_gram()) << 'review'
X = textt.fit_transform(X)
assert X.shape == (25, 116)
mymodel = LogisticRegressionBinaryClassifier().fit(X, y, verbose=0)
X_test = textt.transform(test_reviews)
scores = mymodel.predict(textt.transform(test_reviews))
# View the scores
assert scores.shape == (10, )
assert X_test.shape[0] == 10
def test_ngramfeaturizer_syntax_dict(self):
train_reviews = pandas.DataFrame(data=dict(
review=[
"This is great", "I hate it", "Love it", "Do not like it",
"Really like it", "I hate it", "I like it a lot",
"I kind of hate it", "I do like it", "I really hate it",
"It is very good", "I hate it a bunch", "I love it a bunch",
"I hate it", "I like it very much", "I hate it very much.",
"I really do love it", "I really do hate it", "Love it!",
"Hate it!", "I love it", "I hate it", "I love it", "I hate it",
"I love it"
],
like=[
True, False, True, False, True, False, True, False, True,
False, True, False, True, False, True, False, True, False,
True, False, True, False, True, False, True
]))
test_reviews = pandas.DataFrame(data=dict(review=[
"This is great", "I hate it", "Love it", "Really like it",
"I hate it", "I like it a lot", "I love it", "I do like it",
"I really hate it", "I love it"
]))
y = train_reviews['like']
X = train_reviews.loc[:, train_reviews.columns != 'like']
textt = NGramFeaturizer(word_feature_extractor=n_gram()) << {
'outg': ['review']
}
X = textt.fit_transform(X)
assert X.shape == (25, 117)
# columns ordering changed between 0.22 and 0.23
assert 'review' in (X.columns[0], X.columns[-1])
X = X.drop('review', axis=1)
mymodel = LogisticRegressionBinaryClassifier().fit(X, y, verbose=0)
X_test = textt.transform(test_reviews)
X_test = X_test.drop('review', axis=1)
scores = mymodel.predict(X_test)
# View the scores
assert scores.shape == (10, )
data=dict(
review=[
"This is great",
"I hate it",
"Love it",
"Really like it",
"I hate it",
"I like it a lot",
"I love it",
"I do like it",
"I really hate it",
"I love it"]))
y = train_reviews['like']
X = train_reviews.loc[:, train_reviews.columns != 'like']
ngram = NGramFeaturizer(word_feature_extractor=n_gram()) << 'review'
X = ngram.fit_transform(X)
# view the transformed numerical values and column names
# print(X.head())
mymodel = LogisticRegressionBinaryClassifier().fit(X, y)
X_test = ngram.transform(test_reviews)
scores = mymodel.predict(ngram.transform(test_reviews))
# view the scores
# print(scores.head())