def test_with_numpy_array(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) X_numpy = X.values.copy() with warnings.catch_warnings(record=True) as w: imputer.fit(X) self.assertEqual(len(w), 1) self.assertEqual( "The columns ['NaNs' 'PINF' 'NINF'] did not have any finite values. Filling with zeros.", str(w[0].message)) selected_X = imputer.transform(X) # re-initialize for new dicts imputer = PerColumnImputer() with warnings.catch_warnings(record=True) as w: imputer.fit(X_numpy) self.assertEqual(len(w), 1) self.assertEqual( "The columns [0 1 2] did not have any finite values. Filling with zeros.", str(w[0].message)) selected_X_numpy = imputer.transform(X_numpy) npt.assert_array_equal(selected_X.values, selected_X_numpy.values) self.assertTrue(selected_X_numpy.shape, (1, 100))
def test_standard_replacement_behavior(self): imputer = PerColumnImputer() data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_standard_replacement_behavior(self): imputer = PerColumnImputer() data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_partial_preset_col_to_PINF_given(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) col_to_max = {"a": 100} imputer = PerColumnImputer(col_to_PINF_repl_preset=col_to_max) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_with_numpy_array(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) X_numpy = X.values imputer.fit(X) selected_X = imputer.transform(X) # re-initialize for new dicts imputer = PerColumnImputer() imputer.fit(X_numpy) selected_X_numpy = imputer.transform(X_numpy) npt.assert_array_equal(selected_X.values, selected_X_numpy.values) self.assertTrue(selected_X_numpy.shape, (1, 100))
def test_only_nans_and_infs(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) imputer.fit(X) selected_X = imputer.transform(X) self.assertTrue((selected_X.values == 0).all())
def test_partial_preset_col_to_PINF_given(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) col_to_max = {"a": 100} imputer = PerColumnImputer(col_to_PINF_repl_preset=col_to_max) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_with_numpy_array(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) X_numpy = X.values imputer.fit(X) selected_X = imputer.transform(X) #re-initialize for new dicts imputer = PerColumnImputer() imputer.fit(X_numpy) selected_X_numpy = imputer.transform(X_numpy) npt.assert_array_equal(selected_X.values, selected_X_numpy.values) self.assertTrue(selected_X_numpy.shape, (1, 100))
def test_only_nans_and_infs(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) imputer.fit(X) selected_X = imputer.transform(X) self.assertTrue((selected_X.values == 0).all())
def test_only_subset_of_columns_given(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth_a = [-100.0, 100.0, 0.0, 100.0, -100.0, 1.0, 1.0] truth_b = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data, "b": data}) true_X = pd.DataFrame({"a": truth_a, "b": truth_b}) col_to_median = {"a": 0} imputer = PerColumnImputer(col_to_NAN_repl_preset=col_to_median) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_preset_has_higher_priority_than_fit(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 0.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) col_to_median = {"a": 0} imputer = PerColumnImputer(col_to_NAN_repl_preset=col_to_median) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_only_subset_of_columns_given(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth_a = [-100.0, 100.0, 0.0, 100.0, -100.0, 1.0, 1.0] truth_b = [-100.0, 100.0, 1.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data, "b":data}) true_X = pd.DataFrame({"a":truth_a, "b":truth_b}) col_to_median = {"a": 0} imputer = PerColumnImputer(col_to_NAN_repl_preset=col_to_median) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X,true_X)
def test_preset_has_higher_priority_than_fit(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] truth = [-100.0, 100.0, 0.0, 100.0, -100.0, 1.0, 1.0] X = pd.DataFrame({"a": data}) true_X = pd.DataFrame({"a": truth}) col_to_median = {"a": 0} imputer = PerColumnImputer(col_to_NAN_repl_preset=col_to_median) imputer.fit(X) selected_X = imputer.transform(X) pdt.assert_frame_equal(selected_X, true_X)
def test_only_parameters_of_last_fit_count(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] data_2 = [np.NINF, np.PINF, np.nan, 10.0, -10.0, 3.0, 3.0] truth_a = [-10.0, 10.0, 3.0, 10.0, -10.0, 3.0, 3.0] truth_b = [-10.0, 10.0, 3.0, 10.0, -10.0, 3.0, 3.0] X = pd.DataFrame({"a": data, "b": data}) X_2 = pd.DataFrame({"a": data_2, "b": data_2}) true_X = pd.DataFrame({"a": truth_a, "b": truth_b}) imputer = PerColumnImputer() imputer.fit(X) imputer.fit(X_2) selected_X = imputer.transform(X_2) pdt.assert_frame_equal(selected_X, true_X)
def test_only_parameters_of_last_fit_count(self): data = [np.NINF, np.PINF, np.nan, 100.0, -100.0, 1.0, 1.0] data_2 = [np.NINF, np.PINF, np.nan, 10.0, -10.0, 3.0, 3.0] truth_a = [-10.0, 10.0, 3.0, 10.0, -10.0, 3.0, 3.0] truth_b = [-10.0, 10.0, 3.0, 10.0, -10.0, 3.0, 3.0] X = pd.DataFrame({"a": data, "b": data}) X_2 = pd.DataFrame({"a": data_2, "b": data_2}) true_X = pd.DataFrame({"a": truth_a, "b": truth_b}) imputer = PerColumnImputer() imputer.fit(X) imputer.fit(X_2) selected_X = imputer.transform(X_2) pdt.assert_frame_equal(selected_X, true_X)
def test_only_nans_and_infs(self): imputer = PerColumnImputer() X = pd.DataFrame(index=list(range(100))) X["NaNs"] = np.nan * np.ones(100) X["PINF"] = np.PINF * np.ones(100) X["NINF"] = np.NINF * np.ones(100) with warnings.catch_warnings(record=True) as w: imputer.fit(X) self.assertEqual(len(w), 1) self.assertEqual( "The columns ['NaNs' 'PINF' 'NINF'] did not have any finite values. Filling with zeros.", str(w[0].message)) selected_X = imputer.transform(X) self.assertTrue((selected_X.values == 0).all())