def test_validate_df():
"""Tests the validate_df() function in generate.py."""
# Check that the min_row_ct and min_col_ct arguments work properly.
normal2x2DF = DataFrame({"x": [1, 2], "y": [3, 4]}, index=["a", "b"])
validate_df(normal2x2DF, "Normal 2x2 DF", 0, 0)
validate_df(normal2x2DF, "Normal 2x2 DF", 1, 1)
validate_df(normal2x2DF, "Normal 2x2 DF", 2, 2)
with pytest.raises(ValueError):
validate_df(normal2x2DF, "Normal 2x2 DF", 2, 3)
with pytest.raises(ValueError):
validate_df(normal2x2DF, "Normal 2x2 DF", 2, 4)
with pytest.raises(ValueError):
validate_df(normal2x2DF, "Normal 2x2 DF", 3, 2)
with pytest.raises(ValueError):
validate_df(normal2x2DF, "Normal 2x2 DF", 4, 2)
# Test that the ensure_df_headers_unique() call works. We don't go as in
# depth here as we do above, but we do still want to make sure that it
# works as expected.
# This is just df_bad from test_ensure_df_headers_unique().
nonuniqueRowDF = DataFrame({
"col1": [1, 2, 3],
"col2": [6, 7, 8]
},
index=["a", "b", "a"])
with pytest.raises(ValueError):
validate_df(nonuniqueRowDF, "Non-unique-row DF", 3, 2)
# Check that errors don't "cancel out" (obviously shouldn't be the case,
# but might as well be safe)
with pytest.raises(ValueError):
validate_df(nonuniqueRowDF, "Non-unique-row DF", 3, 3)
# This is just df_bad4 from test_ensure_df_headers_unique().
nonuniqueColDF = DataFrame(
[[1, 6], [2, 7], [3, 8]],
columns=["col1", "col1"],
index=["a", "b", "c"],
)
with pytest.raises(ValueError):
validate_df(nonuniqueColDF, "Non-unique-column DF", 3, 2)
# This is just df_bad5 from test_ensure_df_headers_unique().
nonuniqueColRowDF = DataFrame(
[[1, 6], [2, 7], [3, 8]],
columns=["col1", "col1"],
index=["a", "b", "a"],
)
with pytest.raises(ValueError):
validate_df(nonuniqueColRowDF, "Non-unique-column-and-row DF", 3, 2)
def process_input(
feature_ranks,
sample_metadata,
biom_table,
feature_metadata=None,
extreme_feature_count=None,
):
"""Validates/processes the input files and parameter(s) to Qurro.
In particular, this function
1. Calls validate_df() and then check_column_names() on all of the
input DataFrames passed (feature ranks, sample metadata, feature
metadata if passed).
2. Calls replace_nan() on the metadata DataFrame(s), so that all
missing values are represented consistently with a None (which
will be represented as a null in JSON/JavaScript).
3. Converts the BIOM table to a SparseDataFrame by calling
biom_table_to_sparse_df().
4. Matches up the table with the feature ranks and sample metadata by
calling match_table_and_data().
5. Calls filter_unextreme_features() using the provided
extreme_feature_count. (If it's None, then nothing will be done.)
6. Calls remove_empty_samples_and_features() to filter empty samples
(and features). This is purposefully done *after*
filter_unextreme_features() is called.
7. Calls merge_feature_metadata() on the feature ranks and feature
metadata. (If feature metadata is None, nothing will be done.)
Returns
-------
output_metadata: pd.DataFrame
Sample metadata, but matched with the table and with empty samples
removed.
output_ranks: pd.DataFrame
Feature ranks, post-filtering and with feature metadata columns
added in.
ranking_ids
The ranking columns' names in output_ranks.
feature_metadata_cols: list
The feature metadata columns' names in output_ranks.
output_table: pd.SparseDataFrame
The BIOM table, post matching with the feature ranks and sample
metadata and with empty samples removed.
"""
logging.debug("Starting processing input.")
validate_df(feature_ranks, "feature ranks", 2, 1)
validate_df(sample_metadata, "sample metadata", 1, 1)
if feature_metadata is not None:
# It's cool if there aren't any features actually described in the
# feature metadata (hence why we pass in 0 as the minimum # of rows in
# the feature metadata DataFrame), but we still pass it to
# validate_df() in order to ensure that:
# 1) there's at least one feature metadata column (because
# otherwise the feature metadata is useless)
# 2) column names are unique
validate_df(feature_metadata, "feature metadata", 0, 1)
check_column_names(sample_metadata, feature_ranks, feature_metadata)
# Replace NaN values (which both _metadata_utils.read_metadata_file() and
# qiime2.Metadata use to represent missing values, i.e. ""s) with None --
# this is generally easier for us to handle in the JS side of things (since
# it'll just be consistently converted to null by json.dumps()).
sample_metadata = replace_nan(sample_metadata)
if feature_metadata is not None:
feature_metadata = replace_nan(feature_metadata)
table = biom_table_to_sparse_df(biom_table)
# Match up the table with the feature ranks and sample metadata.
m_table, m_sample_metadata = match_table_and_data(table, feature_ranks,
sample_metadata)
# Note that although we always call filter_unextreme_features(), filtering
# isn't necessarily always done (whether or not depends on the value of
# extreme_feature_count and the contents of the table/ranks).
filtered_table, filtered_ranks = filter_unextreme_features(
m_table, feature_ranks, extreme_feature_count)
# Filter now-empty samples (and empty features) from the BIOM table.
output_table, output_metadata, u_ranks = remove_empty_samples_and_features(
filtered_table, m_sample_metadata, filtered_ranks)
# Save a list of ranking IDs (before we add in feature metadata)
# TODO: just have merge_feature_metadata() give us this?
ranking_ids = u_ranks.columns
output_ranks, feature_metadata_cols = merge_feature_metadata(
u_ranks, feature_metadata)
logging.debug("Finished input processing.")
return (
output_metadata,
output_ranks,
ranking_ids,
feature_metadata_cols,
output_table,
)