def test_text_special_feature_generator(generator_helper, data_helper):
# Given
input_data = data_helper.generate_multi_feature_full()
generator = TextSpecialFeatureGenerator(min_occur_ratio=0,
min_occur_offset=0)
expected_feature_metadata_in_full = {
('object', ('text', )): ['text'],
}
expected_feature_metadata_full = {
('int', ('binned', 'text_special')): [
'text.char_count', 'text.word_count', 'text.capital_ratio',
'text.lower_ratio', 'text.special_ratio', 'text.symbol_ratio. '
]
}
expected_output_data_feat_lower_ratio = [3, 2, 0, 3, 3, 3, 3, 3, 1]
# When
output_data = generator_helper.fit_transform_assert(
input_data=input_data,
generator=generator,
expected_feature_metadata_in_full=expected_feature_metadata_in_full,
expected_feature_metadata_full=expected_feature_metadata_full,
)
assert expected_output_data_feat_lower_ratio == list(
output_data['text.lower_ratio'].values)
def test_text_special_feature_generator_categorical_nan(
generator_helper, data_helper):
# Given
input_data = data_helper.generate_multi_feature_full()
input_data.loc[2, 'text'] = None
input_data['text'] = input_data['text'].astype('category')
type_map_raw = {
'int': 'int',
'float': 'float',
'obj': 'object',
'cat': 'category',
'datetime': 'datetime',
'text': 'category',
'datetime_as_object': 'object',
}
type_map_special = {
'text': ['text'],
}
feature_metadata = FeatureMetadata(
type_map_raw,
type_map_special=type_map_special,
)
generator = TextSpecialFeatureGenerator(min_occur_ratio=0,
min_occur_offset=0)
expected_feature_metadata_in_full = {
('category', ('text', )): ['text'],
}
expected_output_data_feat_lower_ratio = [2, 1, 2, 2, 2, 2, 2, 2, 0]
# When
output_data = generator_helper.fit_transform_assert(
input_data=input_data,
generator=generator,
feature_metadata_in=feature_metadata,
expected_feature_metadata_in_full=expected_feature_metadata_in_full,
expected_feature_metadata_full=expected_feature_metadata_full,
)
assert expected_output_data_feat_lower_ratio == list(
output_data['text.lower_ratio'].values)
def test_pipeline_feature_generator(generator_helper, data_helper):
# Given
input_data = data_helper.generate_multi_feature_full()
toy_vectorizer = CountVectorizer(min_df=2,
ngram_range=(1, 3),
max_features=10,
dtype=np.uint8)
text_ngram_feature_generator = TextNgramFeatureGenerator(
vectorizer=toy_vectorizer)
text_ngram_feature_generator.max_memory_ratio = None # Necessary in test to avoid CI non-deterministically pruning ngram counts.
generator = PipelineFeatureGenerator(generators=[[
IdentityFeatureGenerator(infer_features_in_args=dict(
valid_raw_types=[R_INT, R_FLOAT])),
CategoryFeatureGenerator(),
DatetimeFeatureGenerator(),
TextSpecialFeatureGenerator(),
text_ngram_feature_generator,
]])
expected_feature_metadata_in_full = {
('category', ()): ['cat'],
('datetime', ()): ['datetime'],
('float', ()): ['float'],
('int', ()): ['int'],
('object', ()): ['obj'],
('object', ('datetime_as_object', )): ['datetime_as_object'],
('object', ('text', )): ['text']
}
expected_feature_metadata_full = {
('category', ()): ['obj', 'cat'],
('float', ()): ['float'],
('int', ()): ['int'],
('int', ('binned', 'text_special')): [
'text.char_count', 'text.word_count', 'text.lower_ratio',
'text.special_ratio', 'text.symbol_ratio. '
],
('int', ('datetime_as_int', )): ['datetime', 'datetime_as_object'],
('int', ('text_ngram', )): [
'__nlp__.breaks', '__nlp__.end', '__nlp__.end of',
'__nlp__.end of the', '__nlp__.of', '__nlp__.sentence',
'__nlp__.sentence breaks', '__nlp__.the', '__nlp__.the end',
'__nlp__.world', '__nlp__._total_'
]
}
expected_output_data_feat_datetime = [
1533140820000000000, 1301322000000000000, 1301322000000000000,
1524238620000000000, 1524238620000000000, -5364662400000000000,
7289654340000000000, 1301322000000000000, 1301322000000000000
]
expected_output_data_feat_lower_ratio = [3, 2, 0, 3, 3, 3, 3, 3, 1]
expected_output_data_feat_total = [1, 3, 0, 0, 7, 1, 3, 7, 3]
# When
output_data = generator_helper.fit_transform_assert(
input_data=input_data,
generator=generator,
expected_feature_metadata_in_full=expected_feature_metadata_in_full,
expected_feature_metadata_full=expected_feature_metadata_full,
)
# int and float checks
assert output_data['int'].equals(input_data['int'])
assert output_data['float'].equals(input_data['float'])
# object and category checks
assert list(
output_data['obj'].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]
assert list(output_data['cat'].values) == [
0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan
]
# datetime checks
assert list(output_data['datetime'].values) == list(
output_data['datetime_as_object'].values)
assert expected_output_data_feat_datetime == list(
output_data['datetime'].values)
# text_special checks
assert expected_output_data_feat_lower_ratio == list(
output_data['text.lower_ratio'].values)
# text_ngram checks
assert expected_output_data_feat_total == list(
output_data['__nlp__._total_'].values)
def test_pipeline_feature_generator_removal_advanced(generator_helper,
data_helper):
# Given
input_data = data_helper.generate_multi_feature_full()
toy_vectorizer = CountVectorizer(min_df=2,
ngram_range=(1, 3),
max_features=10,
dtype=np.uint8)
text_ngram_feature_generator = TextNgramFeatureGenerator(
vectorizer=toy_vectorizer)
text_ngram_feature_generator.max_memory_ratio = None # Necessary in test to avoid CI non-deterministically pruning ngram counts.
generator = PipelineFeatureGenerator(generators=[
[
IdentityFeatureGenerator(infer_features_in_args=dict(
valid_raw_types=[R_INT, R_FLOAT])),
CategoryFeatureGenerator(),
DatetimeFeatureGenerator(),
TextSpecialFeatureGenerator(),
text_ngram_feature_generator,
],
[
IdentityFeatureGenerator(infer_features_in_args=dict(
valid_raw_types=[R_CATEGORY]))
],
])
expected_feature_metadata_in_full = {
('category', ()): ['cat'],
('object', ()): ['obj']
}
expected_feature_metadata_full = {('category', ()): ['obj', 'cat']}
expected_feature_metadata_in_unused_full = {
'datetime': ('datetime', ()),
'datetime_as_object': ('object', ('datetime_as_object', )),
'float': ('float', ()),
'int': ('int', ()),
'text': ('object', ('text', ))
}
# When
output_data = generator_helper.fit_transform_assert(
input_data=input_data,
generator=generator,
expected_feature_metadata_in_full=expected_feature_metadata_in_full,
expected_feature_metadata_full=expected_feature_metadata_full,
)
feature_metadata_in_unused_full = generator._feature_metadata_in_unused.to_dict(
)
# object and category checks
assert list(
output_data['obj'].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]
assert list(output_data['cat'].values) == [
0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan
]
assert feature_metadata_in_unused_full == expected_feature_metadata_in_unused_full