def test_feature_select_blocklist(): """ Testing feature_select and get_na_columns pycytominer function """ data_blocklist_df = pd.DataFrame({ "Nuclei_Correlation_Manders_AGP_DNA": [1, 3, 8, 5, 2, 2], "y": [1, 2, 8, 5, 2, 1], "Nuclei_Correlation_RWC_ER_RNA": [9, 3, 8, 9, 2, 9], "zz": [0, -3, 8, 9, 6, 9], }).reset_index(drop=True) result = feature_select(data_blocklist_df, features="infer", operation="blocklist") expected_result = pd.DataFrame({ "y": [1, 2, 8, 5, 2, 1], "zz": [0, -3, 8, 9, 6, 9] }) pd.testing.assert_frame_equal(result, expected_result) result = feature_select( data_blocklist_df, features=data_blocklist_df.columns.tolist(), operation="blocklist", ) expected_result = pd.DataFrame({ "y": [1, 2, 8, 5, 2, 1], "zz": [0, -3, 8, 9, 6, 9] }) pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_get_na_columns(): """ Testing feature_select and get_na_columns pycytominer function """ result = feature_select(data_na_df, features=data_na_df.columns.tolist(), operation="drop_na_columns") expected_result = pd.DataFrame({"yy": [1, 2, 8, 10, 2, 100]}) pd.testing.assert_frame_equal(result, expected_result) result = feature_select( data_na_df, features=data_na_df.columns.tolist(), operation="drop_na_columns", na_cutoff=1, ) pd.testing.assert_frame_equal(result, data_na_df) result = feature_select( data_na_df, features=data_na_df.columns.tolist(), operation="drop_na_columns", na_cutoff=0.3, ) expected_result = pd.DataFrame({ "x": [np.nan, 3, 8, 5, 2, 2], "xx": [np.nan, 3, 8, 5, 2, 2], "yy": [1, 2, 8, 10, 2, 100], "z": [9, 3, 8, 9, 2, np.nan], }) pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_get_na_columns_feature_infer(): """ Testing feature_select and get_na_columns pycytominer function """ result = feature_select( data_feature_infer_df, features="infer", operation="drop_na_columns", na_cutoff=0.3, ) expected_result = pd.DataFrame({ "Metadata_x": [np.nan, np.nan, 8, np.nan, 2, np.nan], "Cytoplasm_xx": [np.nan, 3, 8, 5, 2, 2], "Nuclei_yy": [1, 2, 8, 10, 2, 100], "Cytoplasm_z": [9, 3, 8, 9, 2, np.nan], }) pd.testing.assert_frame_equal(result, expected_result) result = feature_select( data_feature_infer_df, features=data_feature_infer_df.columns.tolist(), operation="drop_na_columns", na_cutoff=0.3, ) expected_result = pd.DataFrame({ "Cytoplasm_xx": [np.nan, 3, 8, 5, 2, 2], "Nuclei_yy": [1, 2, 8, 10, 2, 100], "Cytoplasm_z": [9, 3, 8, 9, 2, np.nan], }) pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_all(): data_all_test_df = data_unique_test_df.assign(zz=a_feature) data_all_test_df.iloc[1, 4] = 2 data_all_test_df.iloc[[x for x in range(0, 50)], 1] = np.nan result = feature_select( profiles=data_all_test_df, features=data_all_test_df.columns.tolist(), operation=["drop_na_columns", "correlation_threshold"], corr_threshold=0.7, ) expected_result = pd.DataFrame({ "c": c_feature, "d": d_feature, "zz": a_feature }).reset_index(drop=True) expected_result.iloc[1, 2] = 2 pd.testing.assert_frame_equal(result, expected_result) # Get temporary directory tmpdir = tempfile.gettempdir() # Write file to output data_file = os.path.join(tmpdir, "test_feature_select.csv") data_all_test_df.to_csv(data_file, index=False, sep=",") out_file = os.path.join(tmpdir, "test_feature_select_out.csv") _ = feature_select( profiles=data_file, features=data_all_test_df.columns.tolist(), operation=["drop_na_columns", "correlation_threshold"], corr_threshold=0.7, output_file=out_file, ) from_file_result = pd.read_csv(out_file) pd.testing.assert_frame_equal(from_file_result, expected_result) result = feature_select( profiles=data_all_test_df, features=data_all_test_df.columns.tolist(), operation=[ "drop_na_columns", "correlation_threshold", "variance_threshold" ], corr_threshold=0.7, ) expected_result = pd.DataFrame({ "c": c_feature, "d": d_feature }).reset_index(drop=True) pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_variance_threshold(): """ Testing feature_select and variance_threshold pycytominer function """ result = feature_select( data_unique_test_df, features=data_unique_test_df.columns.tolist(), operation="variance_threshold", unique_cut=0.01, ) expected_result = pd.DataFrame({ "b": b_feature, "c": c_feature, "d": d_feature }).reset_index(drop=True) pd.testing.assert_frame_equal(result, expected_result) na_data_unique_test_df = data_unique_test_df.copy() na_data_unique_test_df.iloc[[x for x in range(0, 50)], 1] = np.nan result = feature_select( na_data_unique_test_df, features=na_data_unique_test_df.columns.tolist(), operation=["drop_na_columns", "variance_threshold"], ) expected_result = pd.DataFrame({ "c": c_feature, "d": d_feature }).reset_index(drop=True) pd.testing.assert_frame_equal(result, expected_result) na_data_unique_test_df = data_unique_test_df.copy() na_data_unique_test_df.iloc[[x for x in range(0, 50)], 1] = np.nan result = feature_select( na_data_unique_test_df, features=na_data_unique_test_df.columns.tolist(), operation=["variance_threshold", "drop_na_columns"], ) expected_result = pd.DataFrame({ "c": c_feature, "d": d_feature }).reset_index(drop=True) pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_drop_outlier(): """ Testing feature_select and get_na_columns pycytominer function """ result = feature_select( data_outlier_df, features="infer", operation="drop_outliers" ) expected_result = data_outlier_df.drop(["Cells_zz", "Nuclei_z"], axis="columns") pd.testing.assert_frame_equal(result, expected_result) result = feature_select( data_outlier_df, features="infer", operation="drop_outliers", outlier_cutoff=30 ) expected_result = data_outlier_df.drop(["Cells_zz"], axis="columns") pd.testing.assert_frame_equal(result, expected_result) result = feature_select( data_outlier_df, features=["Cells_x", "Cytoplasm_y"], operation="drop_outliers" ) pd.testing.assert_frame_equal(result, data_outlier_df)
def test_feature_select_correlation_threshold(): """ Testing feature_select and correlation_threshold pycytominer function """ result = feature_select(data_df, features=data_df.columns.tolist(), operation="correlation_threshold") expected_result = data_df.drop(["y"], axis="columns") pd.testing.assert_frame_equal(result, expected_result) data_cor_thresh_na_df = data_df.copy() data_cor_thresh_na_df.iloc[0, 2] = np.nan result = feature_select( data_cor_thresh_na_df, features=data_cor_thresh_na_df.columns.tolist(), operation=["drop_na_columns", "correlation_threshold"], ) expected_result = data_df.drop(["z", "x"], axis="columns") pd.testing.assert_frame_equal(result, expected_result)
def test_feature_select_compress(): compress_file = os.path.join(tmpdir, "test_feature_select_compress.csv.gz") _ = feature_select( data_na_df, features=data_na_df.columns.tolist(), operation="drop_na_columns", output_file=compress_file, compression="gzip", ) expected_result = pd.DataFrame({"yy": [1, 2, 8, 10, 2, 100]}) result = pd.read_csv(compress_file) pd.testing.assert_frame_equal(result, expected_result)
# ## Perform Feature Selection # In[3]: eb = ("actin", "DNA", "dist", "nuclear") features = [x for x in train_df.columns if x.startswith(eb)] # In[4]: train_feature_select_df = feature_select(profiles=train_df, features=features, operation=[ "drop_na_columns", "variance_threshold", "correlation_threshold" ], na_cutoff=0.01, corr_threshold=0.95, corr_method="pearson", freq_cut=0.01, unique_cut=0.001) # In[5]: selected_features = [ x for x in train_feature_select_df.columns if x.startswith(eb) ] print(train_feature_select_df.shape) train_feature_select_df.head()