def test_sparse_is_equivalent(self):
random.seed("the man who sold the world")
X = list(self._get_random_tuples())
# fit + transform
A = FeatureMappingFlattener(sparse=True)
YA = A.fit_transform(X).todense()
# fit_transform
B = FeatureMappingFlattener(sparse=False)
YB = B.fit_transform(X)
self.assertTrue(numpy.array_equal(YA, YB))
def test_sparse_is_equivalent(self):
random.seed("the man who sold the world")
X = list(self._get_random_tuples())
# fit + transform
A = FeatureMappingFlattener(sparse=True)
YA = A.fit_transform(X).todense()
# fit_transform
B = FeatureMappingFlattener(sparse=False)
YB = B.fit_transform(X)
self.assertTrue(numpy.array_equal(YA, YB))
def test_fit_transform_consumes_data_only_once(self):
random.seed("a kiss to build a dream on")
X = list(self._get_random_tuples())
X_consumable = (x for x in X)
V1 = FeatureMappingFlattener(sparse=False)
V1.fit(X)
Z1 = V1.transform(X)
Z2 = V1.fit_transform(X_consumable)
self.assertTrue(numpy.array_equal(Z1, Z2))
def test_fit_transform_consumes_data_only_once(self):
random.seed("a kiss to build a dream on")
X = list(self._get_random_tuples())
X_consumable = (x for x in X)
V1 = FeatureMappingFlattener(sparse=False)
V1.fit(X)
Z1 = V1.transform(X)
Z2 = V1.fit_transform(X_consumable)
self.assertTrue(numpy.array_equal(Z1, Z2))
def test_fit_transform_ok(self):
random.seed("a kiss to build a dream on")
X = list(self._get_random_tuples())
for sparse in [True, False]:
V = FeatureMappingFlattener(sparse=sparse)
Z = V.fit_transform(X)
n = 100
m = 4 + 3 + 5 # 4 float, 1 enum, 1 list
self.assertEqual(Z.shape, (n, m))
d = next(self._get_random_tuples())
Z = V.transform([d]) # Test that works for one dict too
self.assertEqual(Z.shape, (1, m))
def test_fit_transform_ok(self):
random.seed("a kiss to build a dream on")
X = list(self._get_random_tuples())
for sparse in [True, False]:
V = FeatureMappingFlattener(sparse=sparse)
Z = V.fit_transform(X)
n = 100
m = 4 + 3 + 5 # 4 float, 1 enum, 1 list
self.assertEqual(Z.shape, (n, m))
d = next(self._get_random_tuples())
Z = V.transform([d]) # Test that works for one dict too
self.assertEqual(Z.shape, (1, m))
def test_fit_transform_equivalent(self):
random.seed("j0hny guitar")
X = list(self._get_random_tuples())
for sparse in [True, False]:
# fit + transform
A = FeatureMappingFlattener(sparse=sparse)
A.fit(X)
YA = A.transform(X)
# fit_transform
B = FeatureMappingFlattener(sparse=sparse)
YB = B.fit_transform(X)
if sparse:
self.assertTrue(numpy.array_equal(YA.todense(), YB.todense()))
else:
self.assertTrue(numpy.array_equal(YA, YB))
self.assertEqual(A.indexes, B.indexes)
self.assertEqual(A.reverse, B.reverse)
def test_fit_transform_equivalent(self):
random.seed("j0hny guitar")
X = list(self._get_random_tuples())
for sparse in [True, False]:
# fit + transform
A = FeatureMappingFlattener(sparse=sparse)
A.fit(X)
YA = A.transform(X)
# fit_transform
B = FeatureMappingFlattener(sparse=sparse)
YB = B.fit_transform(X)
if sparse:
self.assertTrue(numpy.array_equal(YA.todense(), YB.todense()))
else:
self.assertTrue(numpy.array_equal(YA, YB))
self.assertEqual(A.indexes, B.indexes)
self.assertEqual(A.reverse, B.reverse)
class Vectorizer(object):
"""
Vectorizer(features) provides a scikit-learn compatible component that
given a collection of data points turns it into a matrix of vectors,
where each vector contains the evaluation of every given feature for a
single data point.
Numerical features are mapped to a column of the resulting matrix.
Enumerated features are mapped to multiple columns (one for each possible
enumerated value), using 0 or 1 to indicate the presence of the enumerated
value. Vectorial features are mapped to multiple columns.
The API of this class follows scikit-learn conventions, see
http://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html
Vectorizer(features, tolerant=True) changes the feature evaluation strategy
to one that is more tolerant to failures when evaluating features. It is
useful for testing and to run rough experiments when you still aren't sure
if your data is clean or your features are correct. See the documentation
for featureforge.evaluator.TolerantFeatureEvaluator
Vectorizer(features, sparse=True) changes the result data type, returning a
sparse numpy matrix instead of a dense matrix. See the documentation on
featureforge.flattener.Flattener
"""
def __init__(self, features, tolerant=False, sparse=True):
# Upgrade `features` to `Feature` instances.
features = list(map(make_feature, features))
if tolerant:
self.evaluator = TolerantFeatureEvaluator(features)
else:
self.evaluator = FeatureEvaluator(features)
self.flattener = FeatureMappingFlattener(sparse=sparse)
def fit(self, X, y=None):
Xt = self.evaluator.fit_transform(X, y)
self.flattener.fit(Xt, y)
return self
def fit_transform(self, X, y):
Xt = self.evaluator.fit_transform(X, y)
return self.flattener.fit_transform(Xt, y)
def transform(self, X):
Xt = self.evaluator.transform(X)
return self.flattener.transform(Xt)
def column_to_feature(self, i):
"""
Given a column index in the vectorizer's output matrix it returns the
feature that originates that column.
The return value is a tuple (feature, value).
`feature` is the feature given in the initialization and `value`
depends on the kind of feature that the column represents:
- If the feature spawns numbers then `value` is `None` and should
be ignored.
- If the feature spawns strings then `value` is the string that
corresponds to the one-hot encoding of that column.
- If the feature spawns an array then `value` is the index within
the spawned array that corresponds to that column.
"""
j, value = self.flattener.reverse[i]
feature = self.evaluator.alive_features[j]
return feature, value
class Vectorizer(object):
"""
Vectorizer(features) provides a scikit-learn compatible component that
given a collection of data points turns it into a matrix of vectors,
where each vector contains the evaluation of every given feature for a
single data point.
Numerical features are mapped to a column of the resulting matrix.
Enumerated features are mapped to multiple columns (one for each possible
enumerated value), using 0 or 1 to indicate the presence of the enumerated
value. Vectorial features are mapped to multiple columns.
The API of this class follows scikit-learn conventions, see
http://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html
Vectorizer(features, tolerant=True) changes the feature evaluation strategy
to one that is more tolerant to failures when evaluating features. It is
useful for testing and to run rough experiments when you still aren't sure
if your data is clean or your features are correct. See the documentation
for featureforge.evaluator.TolerantFeatureEvaluator
Vectorizer(features, sparse=True) changes the result data type, returning a
sparse numpy matrix instead of a dense matrix. See the documentation on
featureforge.flattener.Flattener
"""
def __init__(self, features, tolerant=False, sparse=True):
# Upgrade `features` to `Feature` instances.
features = list(map(make_feature, features))
if tolerant:
self.evaluator = TolerantFeatureEvaluator(features)
else:
self.evaluator = FeatureEvaluator(features)
self.flattener = FeatureMappingFlattener(sparse=sparse)
def fit(self, X, y=None):
Xt = self.evaluator.fit_transform(X, y)
self.flattener.fit(Xt, y)
return self
def fit_transform(self, X, y=None):
Xt = self.evaluator.fit_transform(X, y)
return self.flattener.fit_transform(Xt, y)
def transform(self, X):
Xt = self.evaluator.transform(X)
return self.flattener.transform(Xt)
def column_to_feature(self, i):
"""
Given a column index in the vectorizer's output matrix it returns the
feature that originates that column.
The return value is a tuple (feature, value).
`feature` is the feature given in the initialization and `value`
depends on the kind of feature that the column represents:
- If the feature spawns numbers then `value` is `None` and should
be ignored.
- If the feature spawns strings then `value` is the string that
corresponds to the one-hot encoding of that column.
- If the feature spawns an array then `value` is the index within
the spawned array that corresponds to that column.
"""
j, value = self.flattener.reverse[i]
feature = self.evaluator.alive_features[j]
return feature, value
