def test_filtering_large_efc():
"""Tests filter_unextreme_features() when (the extreme feature count * 2)
is greater than or equal to the number of ranked features.
"""
table, ranks = get_test_data()
# The number of ranked features is 8.
filtered_table, filtered_ranks = filter_unextreme_features(table, ranks, 4)
assert_frame_equal(table, filtered_table)
assert_frame_equal(ranks, filtered_ranks)
filtered_table, filtered_ranks = filter_unextreme_features(table, ranks, 8)
assert_frame_equal(table, filtered_table)
assert_frame_equal(ranks, filtered_ranks)
def test_filtering_no_efc():
"""Tests filter_unextreme_features() when the extreme feature count is None
(i.e. the user didn't use the -x option, and no filtering should be
done).
"""
table, ranks = get_test_data()
filtered_table, filtered_ranks = filter_unextreme_features(
table, ranks, None)
assert_frame_equal(table, filtered_table)
assert_frame_equal(ranks, filtered_ranks)
def test_filtering_invalid_efc():
"""Tests that filter_unextreme_features() throws an error when the
extreme feature count is less than 1 and/or not an integer.
"""
table, ranks = get_test_data()
with pytest.raises(ValueError):
filter_unextreme_features(table, ranks, 0)
with pytest.raises(ValueError):
filter_unextreme_features(table, ranks, -1)
with pytest.raises(ValueError):
filter_unextreme_features(table, ranks, -2)
with pytest.raises(ValueError):
filter_unextreme_features(table, ranks, 1.5)
with pytest.raises(ValueError):
filter_unextreme_features(table, ranks, 5.5)
def test_filtering_basic():
"""Tests the standard behavior of filter_unextreme_features()."""
table, ranks = get_test_data()
filtered_table, filtered_ranks = filter_unextreme_features(table, ranks, 2)
# Check that the appropriate features/samples were filtered out of the
# table. NOTE -- I know this is sloppy code. Would like to fix it in the
# future.
for fid in ["F1", "F2", "F7", "F8"]:
assert fid in filtered_table.index
for fid in ["F3", "F4", "F5", "F6"]:
assert fid not in filtered_table.index
# Check that all samples were preserved.
# (The removal of empty features is done *after*
# filter_unextreme_features() is called in normal Qurro execution, so we
# should expect all samples -- even empty ones -- to remain here.
for sid in ["S1", "S2", "S3", "S4", "S5"]:
assert sid in filtered_table.columns
# Check that the appropriate data is left in the table.
assert list(filtered_table.loc["F1"]) == [0, 1, 0, 3, 4]
assert list(filtered_table.loc["F2"]) == [5, 6, 0, 8, 9]
assert list(filtered_table.loc["F7"]) == [30, 31, 0, 33, 34]
assert list(filtered_table.loc["F8"]) == [35, 36, 0, 38, 39]
# Check that the rank filtering worked as expected.
expected_filtered_ranks = DataFrame(
{
"Rank 0": [1, 2, 7, 8],
"Rank 1": [8, 7, 2, 1]
},
index=["F1", "F2", "F7", "F8"],
)
assert_frame_equal(filtered_ranks,
expected_filtered_ranks,
check_like=True)
def process_input(
feature_ranks,
sample_metadata,
biom_table,
feature_metadata=None,
extreme_feature_count=None,
):
"""Validates/processes the input files and parameter(s) to Qurro.
In particular, this function
1. Calls validate_df() and then check_column_names() on all of the
input DataFrames passed (feature ranks, sample metadata, feature
metadata if passed).
2. Calls replace_nan() on the metadata DataFrame(s), so that all
missing values are represented consistently with a None (which
will be represented as a null in JSON/JavaScript).
3. Converts the BIOM table to a SparseDataFrame by calling
biom_table_to_sparse_df().
4. Matches up the table with the feature ranks and sample metadata by
calling match_table_and_data().
5. Calls filter_unextreme_features() using the provided
extreme_feature_count. (If it's None, then nothing will be done.)
6. Calls remove_empty_samples_and_features() to filter empty samples
(and features). This is purposefully done *after*
filter_unextreme_features() is called.
7. Calls merge_feature_metadata() on the feature ranks and feature
metadata. (If feature metadata is None, nothing will be done.)
Returns
-------
output_metadata: pd.DataFrame
Sample metadata, but matched with the table and with empty samples
removed.
output_ranks: pd.DataFrame
Feature ranks, post-filtering and with feature metadata columns
added in.
ranking_ids
The ranking columns' names in output_ranks.
feature_metadata_cols: list
The feature metadata columns' names in output_ranks.
output_table: pd.SparseDataFrame
The BIOM table, post matching with the feature ranks and sample
metadata and with empty samples removed.
"""
logging.debug("Starting processing input.")
validate_df(feature_ranks, "feature ranks", 2, 1)
validate_df(sample_metadata, "sample metadata", 1, 1)
if feature_metadata is not None:
# It's cool if there aren't any features actually described in the
# feature metadata (hence why we pass in 0 as the minimum # of rows in
# the feature metadata DataFrame), but we still pass it to
# validate_df() in order to ensure that:
# 1) there's at least one feature metadata column (because
# otherwise the feature metadata is useless)
# 2) column names are unique
validate_df(feature_metadata, "feature metadata", 0, 1)
check_column_names(sample_metadata, feature_ranks, feature_metadata)
# Replace NaN values (which both _metadata_utils.read_metadata_file() and
# qiime2.Metadata use to represent missing values, i.e. ""s) with None --
# this is generally easier for us to handle in the JS side of things (since
# it'll just be consistently converted to null by json.dumps()).
sample_metadata = replace_nan(sample_metadata)
if feature_metadata is not None:
feature_metadata = replace_nan(feature_metadata)
table = biom_table_to_sparse_df(biom_table)
# Match up the table with the feature ranks and sample metadata.
m_table, m_sample_metadata = match_table_and_data(table, feature_ranks,
sample_metadata)
# Note that although we always call filter_unextreme_features(), filtering
# isn't necessarily always done (whether or not depends on the value of
# extreme_feature_count and the contents of the table/ranks).
filtered_table, filtered_ranks = filter_unextreme_features(
m_table, feature_ranks, extreme_feature_count)
# Filter now-empty samples (and empty features) from the BIOM table.
output_table, output_metadata, u_ranks = remove_empty_samples_and_features(
filtered_table, m_sample_metadata, filtered_ranks)
# Save a list of ranking IDs (before we add in feature metadata)
# TODO: just have merge_feature_metadata() give us this?
ranking_ids = u_ranks.columns
output_ranks, feature_metadata_cols = merge_feature_metadata(
u_ranks, feature_metadata)
logging.debug("Finished input processing.")
return (
output_metadata,
output_ranks,
ranking_ids,
feature_metadata_cols,
output_table,
)
