def test_filter(self):
with_nans = pd.DataFrame(
data=dict(Sepal_Length=[2.5, np.nan, 2.1, 1.0],
Sepal_Width=[.75, .9, .8, .76],
Petal_Length=[np.nan, 2.5, 2.6, 2.4],
Petal_Width=[.8, .7, .9, 0.7]))
tmpfile = 'tmpfile_with_nans.csv'
with_nans.to_csv(tmpfile, index=False, na_rep='?')
file_schema = 'sep=, col=Petal_Length:R4:0 col=Petal_Width:R4:1 ' \
'col=Sepal_Length:R4:2 col=Sepal_Width:R4:3 header+'
data = FileDataStream(tmpfile, schema=file_schema)
xf = Filter(columns=[
'Petal_Length', 'Petal_Width', 'Sepal_Length', 'Sepal_Width'
])
features = xf.fit_transform(data)
assert features.shape == (2, 4)
print(features.columns)
# columns ordering changed between 0.22 and 0.23
assert set(features.columns) == {
'Petal_Length', 'Petal_Width', 'Sepal_Length', 'Sepal_Width'
}
os.remove(tmpfile)
def test_inf(self):
data = DataFrame(data=dict(f0=[np.inf, 1, 2, 3, 4, 5, 6],
f1=[1, 2, -np.Infinity, 3, 4, 5, 6]))
xf = Filter(columns=['f0'])
filtered = xf.fit_transform(data)
assert_equal(filtered['f0'][0], np.inf)
assert_equal(filtered['f1'][2], -np.inf)
def test_missing(self):
data = DataFrame(data=dict(f0=[np.nan, 1, 2, 3, 4, 5, 6],
f1=[1, 2, np.nan, 3, 4, 5, 6],
f2=[np.nan, 1, np.nan, 2, 3, np.nan, 4]))
for col in data.columns:
xf = Filter(columns=[col])
filtered = xf.fit_transform(data)
count = [isinstance(x, str) or not isnan(x)
for x in data[col]].count(True)
assert_equal(filtered.shape[0], count)
def test_check_estimator_filter(self):
dataTrain = pd.DataFrame(
data=dict(Sepal_Length=[2.5, np.nan, 2.1, 1.0],
Sepal_Width=[.75, .9, .8, .76],
Petal_Length=[np.nan, 2.5, 2.6, 2.4],
Petal_Width=[.8, .7, .9, 0.7],
Species=["setosa", "virginica", "", 'versicolor']))
filter = Filter() << ["Sepal_Length", "Petal_Length"]
data_idv = filter.fit_transform(dataTrain)
assert data_idv is not None
assert len(data_idv) > 0
###############################################################################
# Filter
import numpy as np
import pandas as pd
from nimbusml import FileDataStream
from nimbusml.preprocessing.missing_values import Filter
with_nans = pd.DataFrame(
data=dict(Sepal_Length=[2.5, np.nan, 2.1, 1.0],
Sepal_Width=[.75, .9, .8, .76],
Petal_Length=[np.nan, 2.5, 2.6, 2.4],
Petal_Width=[.8, .7, .9, 0.7],
Species=["setosa", "viginica", "", 'versicolor']))
# write NaNs to file to see how transforms work
tmpfile = 'tmpfile_with_nans.csv'
with_nans.to_csv(tmpfile, index=False)
# schema for reading directly from text files
schema = 'sep=, col=Petal_Length:R4:0 col=Petal_Width:R4:1' \
'col=Sepal_Length:R4:2 col=Sepal_Width:R4:3 col=Species:TX:4 header+'
data = FileDataStream.read_csv(tmpfile)
print(data.schema)
# filter out rows where Sepal_Length is NaN
nafilter = Filter() << ['Sepal_Length', 'Petal_Length']
print(with_nans)
print('NAFilter\n', nafilter.fit_transform(data))
###############################################################################
# Filter
import numpy as np
import pandas as pd
from nimbusml import FileDataStream
from nimbusml.preprocessing.missing_values import Filter
with_nans = pd.DataFrame(data=dict(Sepal_Length=[2.5, np.nan, 2.1, 1.0],
Sepal_Width=[.75, .9, .8, .76],
Petal_Length=[np.nan, 2.5, 2.6, 2.4],
Petal_Width=[.8, .7, .9, 0.7]))
# write NaNs to file to show how this transform work
tmpfile = 'tmpfile_with_nans.csv'
with_nans.to_csv(tmpfile, index=False)
data = FileDataStream.read_csv(tmpfile, sep=',', numeric_dtype=np.float32)
# transform usage
xf = Filter(
columns=['Petal_Length', 'Petal_Width', 'Sepal_Length', 'Sepal_Width'])
# fit and transform
features = xf.fit_transform(data)
# print features
print(features.head())
# Petal_Length Petal_Width Sepal_Length Sepal_Width
# 0 2.4 0.7 1.0 0.76