def test_group_significance_output_formatter(self):
"""output_formatter works"""
#Using ANOVA test for example
otu_table_1 = """{"id": "None","format": "Biological Observation Matrix 1.0.0","format_url": "http://biom-format.org","type": "OTU table","generated_by": "BIOM-Format 1.1.2","date": "2013-08-16T15:23:02.872397","matrix_type": "sparse","matrix_element_type": "float","shape": [6, 8],"data": [[0,0,28.0],[0,1,52.0],[0,2,51.0],[0,3,78.0],[0,4,16.0],[0,5,77.0],[0,6,73.0],[0,7,6.0],[1,0,25.0],[1,1,14.0],[1,2,11.0],[1,3,32.0],[1,4,48.0],[1,5,63.0],[1,6,27.0],[1,7,38.0],[2,0,31.0],[2,1,2.0],[2,2,15.0],[2,3,69.0],[2,4,64.0],[2,5,27.0],[2,6,64.0],[2,7,54.0],[3,0,36.0],[3,1,68.0],[3,2,70.0],[3,3,65.0],[3,4,33.0],[3,5,62.0],[3,6,60.0],[3,7,23.0],[4,0,16.0],[4,1,41.0],[4,2,59.0],[4,3,40.0],[4,4,15.0],[4,5,3.0],[4,6,35.0],[4,7,5.0],[5,0,32.0],[5,1,8.0],[5,2,54.0],[5,3,98.0],[5,4,29.0],[5,5,50.0],[5,6,93.0],[5,7,19.0]],"rows": [{"id": "OTU1", "metadata": {"taxonomy": "k__One"}},{"id": "OTU2", "metadata": {"taxonomy": "k__Two"}},{"id": "OTU3", "metadata": {"taxonomy": "k__Three"}},{"id": "OTU4", "metadata": {"taxonomy": "k__Four"}},{"id": "OTU5", "metadata": {"taxonomy": "k__Five"}},{"id": "OTU6", "metadata": {"taxonomy": "k__Six"}}],"columns": [{"id": "Sample1", "metadata": null},{"id": "Sample2", "metadata": null},{"id": "Sample3", "metadata": null},{"id": "Sample4", "metadata": null},{"id": "Sample5", "metadata": null},{"id": "Sample6", "metadata": null},{"id": "Sample7", "metadata": null},{"id": "Sample8", "metadata": null}]}"""
bt = parse_biom_table(otu_table_1)
test_stats = [0.18990151199889027,
6.7142857142857144,
1.8424031345232912,
1.4643841007477372,
4.5675332910589734,
9.8389688760617899e-05]
pvals = [0.6782397284636259,
0.041145883121579234,
0.22350244183135481,
0.27174025956151771,
0.076447615888438403,
0.9924073718332751]
means = [[52.25, 43.0],
[20.5, 44.0],
[29.25, 52.25],
[59.75, 44.5],
[39.0, 14.5],
[48.0, 47.75]]
fdr_pvals = [0.8138876741563511,
0.24687529872947539,
0.44700488366270963,
0.40761038934227656,
0.22934284766531521,
0.9924073718332751]
bon_pvals = [4.069438370781755,
0.2468752987294754,
1.3410146509881289,
1.6304415573691062,
0.4586856953306304,
5.95444423099965]
cat_sample_indices = {'cat1': [0, 1, 2, 3], 'cat2': [4, 5, 6, 7]}
line_header_out = 'OTU\tTest-Statistic\tP\tFDR_P\tBonferroni_P\tcat1_mean\tcat2_mean\ttaxonomy'
line_1_out = 'OTU1\t0.189901511999\t0.678239728464\t0.813887674156\t4.06943837078\t52.25\t43.0\tk__One'
line_6_out = 'OTU6\t9.83896887606e-05\t0.992407371833\t0.992407371833\t5.954444231\t48.0\t47.75\tk__Six'
lines = group_significance_output_formatter(bt, test_stats, pvals,
fdr_pvals, bon_pvals, means, cat_sample_indices, md_key='taxonomy')
self.assertEqual(lines[0], line_header_out)
self.assertEqual(lines[1], line_1_out)
self.assertEqual(lines[6], line_6_out)
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
# sync the mapping file and the biom file
tmp_bt = load_table(opts.otu_table_fp)
tmp_pmf, _ = parse_mapping_file_to_dict(opts.mapping_fp)
pmf, bt, nonshared_samples = sync_biom_and_mf(tmp_pmf, tmp_bt)
# test error conditions for overlapping mf and bt
if not opts.biom_samples_are_superset:
# user indicates biom sample should be subset of mapping file samples
if any([i in nonshared_samples for i in tmp_bt.ids()]):
raise ValueError('The samples in the biom table are a superset of' +
' the samples in the mapping file. The script will abort in' +
' this case even though the calculations wouldn\'t be' +
' affected, to ensure consistency within QIIME. Pass the' +
' --biom_samples_are_superset option to disable this behavior.')
# user wants non-overlapping samples printed out
if opts.print_non_overlap:
print 'The following samples were not shared between the mapping file' +\
' and the biom file and will not be included in the analysis:\n' +\
' '.join(nonshared_samples)
# find group indices
sam_cats = get_sample_cats(pmf, opts.category)
cat_sam_groups = get_cat_sample_groups(sam_cats)
cat_sam_indices = get_sample_indices(cat_sam_groups, bt)
# sanity check to prevent inscrutable errors later
if not all([len(v) > 0 for k, v in cat_sam_indices.items()]):
raise ValueError('At least one metadata group has no samples. Check ' +
'that the mapping file has at least one sample for each value in ' +
'the passed category.')
if opts.test in TWO_GROUP_TESTS and len(cat_sam_indices) > 2:
option_parser.error('The t-test and mann_whitney_u test may ' +
'only be used when there are two sample groups. Choose another ' +
'test or another metadata category.')
# check that assumptions are met for a given test:
if opts.test == 'mann_whitney_u':
sams = reduce(lambda x, y: len(x) + len(y), cat_sam_indices.values())
if sams <= 20:
raise ValueError('The number of samples is too small to use the ' +
'Mann-Whitney-U normal approximation. Review the script ' +
'documentation.')
# check that the G-test was not selected if the table appears to be
# relative abundance
if opts.test == 'g_test':
if allclose(bt.sum(axis='sample'), 1.) or (bt.sum(axis='whole') == 1.):
raise ValueError('It appears that the biom table you have passed '
'is a relative abundance table where values i,j (obsevation i '
'count in sample j) are fractional and the sum of the columns '
'is 1.0. This will fail to work properly with the G-test. If '
'your data sums to 1 in each column but your data is not '
'relative abundance then the tests will fail anyway because '
'of the reduced number of observations.')
# run actual tests
data_feed = group_significance_row_generator(bt, cat_sam_indices)
test_stats, pvals, means = run_group_significance_test(
data_feed, opts.test,
GROUP_TEST_CHOICES, int(opts.permutations))
# calculate corrected pvals
fdr_pvals = array(benjamini_hochberg_step_down(pvals))
bon_pvals = bonferroni_correction(pvals)
# correct for cases where values above 1.0 due to correction
fdr_pvals = where(fdr_pvals > 1.0, 1.0, fdr_pvals)
bon_pvals = where(bon_pvals > 1.0, 1.0, bon_pvals)
# write output results after sorting
lines = group_significance_output_formatter(bt, test_stats, pvals,
fdr_pvals, bon_pvals, means, cat_sam_indices, md_key=opts.metadata_key)
lines = sort_by_pval(lines, ind=2)
o = open(opts.output_fp, 'w')
o.writelines('\n'.join(lines))
o.close()
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
# sync the mapping file and the biom file
tmp_bt = load_table(opts.otu_table_fp)
tmp_pmf, _ = parse_mapping_file_to_dict(opts.mapping_fp)
pmf, bt, nonshared_samples = sync_biom_and_mf(tmp_pmf, tmp_bt)
# test error conditions for overlapping mf and bt
if not opts.biom_samples_are_superset:
# user indicates biom sample should be subset of mapping file samples
if any([i in nonshared_samples for i in tmp_bt.ids()]):
raise ValueError(
'The samples in the biom table are a superset of' +
' the samples in the mapping file. The script will abort in' +
' this case even though the calculations wouldn\'t be' +
' affected, to ensure consistency within QIIME. Pass the' +
' --biom_samples_are_superset option to disable this behavior.'
)
# user wants non-overlapping samples printed out
if opts.print_non_overlap:
print 'The following samples were not shared between the mapping file' +\
' and the biom file and will not be included in the analysis:\n' +\
' '.join(nonshared_samples)
# find group indices
sam_cats = get_sample_cats(pmf, opts.category)
cat_sam_groups = get_cat_sample_groups(sam_cats)
cat_sam_indices = get_sample_indices(cat_sam_groups, bt)
# sanity check to prevent inscrutable errors later
if not all([len(v) > 0 for k, v in cat_sam_indices.items()]):
raise ValueError(
'At least one metadata group has no samples. Check ' +
'that the mapping file has at least one sample for each value in '
+ 'the passed category.')
if opts.test in TWO_GROUP_TESTS and len(cat_sam_indices) > 2:
option_parser.error(
'The t-test and mann_whitney_u test may ' +
'only be used when there are two sample groups. Choose another ' +
'test or another metadata category.')
# check that assumptions are met for a given test:
if opts.test == 'mann_whitney_u':
sams = reduce(lambda x, y: len(x) + len(y), cat_sam_indices.values())
if sams <= 20:
raise ValueError(
'The number of samples is too small to use the ' +
'Mann-Whitney-U normal approximation. Review the script ' +
'documentation.')
# check that the G-test was not selected if the table appears to be
# relative abundance
if opts.test == 'g_test':
if allclose(bt.sum(axis='sample'), 1.) or (bt.sum(axis='whole') == 1.):
raise ValueError(
'It appears that the biom table you have passed '
'is a relative abundance table where values i,j (obsevation i '
'count in sample j) are fractional and the sum of the columns '
'is 1.0. This will fail to work properly with the G-test. If '
'your data sums to 1 in each column but your data is not '
'relative abundance then the tests will fail anyway because '
'of the reduced number of observations.')
# run actual tests
data_feed = group_significance_row_generator(bt, cat_sam_indices)
test_stats, pvals, means = run_group_significance_test(
data_feed, opts.test, GROUP_TEST_CHOICES, int(opts.permutations))
# calculate corrected pvals
fdr_pvals = array(benjamini_hochberg_step_down(pvals))
bon_pvals = bonferroni_correction(pvals)
# correct for cases where values above 1.0 due to correction
fdr_pvals = where(fdr_pvals > 1.0, 1.0, fdr_pvals)
bon_pvals = where(bon_pvals > 1.0, 1.0, bon_pvals)
# write output results after sorting
lines = group_significance_output_formatter(bt,
test_stats,
pvals,
fdr_pvals,
bon_pvals,
means,
cat_sam_indices,
md_key=opts.metadata_key)
lines = sort_by_pval(lines, ind=2)
o = open(opts.output_fp, 'w')
o.writelines('\n'.join(lines))
o.close()