def test_convert_taxa_to_list(self):
format_list = ['BOLD', 'REG', 'REG', 'REG', 'REG', 'REG', 'COLOR']
known_latex_format = ['\\begin{itemize}\n\\item \\textbf{cont. '
'Unclassified Kingdom Bacteria}\n\\item cont. '
'Unclassified Phylum Proteobacteria\n\\item '
'Unclassified Class Gammaproteobacteria\n\\item'
' Unclassified Order Enterobacteriales\n\\item'
' Unclassified Family Enterbacteriaceae\n\\item'
' Genus \\textit{Escherichia}\n\\item '
'\\textcolor{red}{\\textit{Escherichia coli}}\n'
'\\end{itemize}']
known_latex_comma = ['\\textbf{cont. Unclassified Kingdom Bacteria},'
' cont. Unclassified Phylum Proteobacteria, '
'Unclassified Class Gammaproteobacteria, '
'Unclassified Order Enterobacteriales, '
'Unclassified Family Enterbacteriaceae, '
'Genus \\textit{Escherichia}, \\textcolor{blue}'
'{\\textit{Escherichia coli}}']
known_raw_format = ['\n o *cont. Unclassified Kingdom Bacteria*\n '
'o cont. Unclassified Phylum Proteobacteria\n '
'o Unclassified Class Gammaproteobacteria\n '
'o Unclassified Order Enterobacteriales\n '
'o Unclassified Family Enterbacteriaceae\n '
'o Genus Escherichia\n '
'o *Escherichia coli*']
known_raw_comma = ['*cont. Unclassified Kingdom Bacteria*,'
' cont. Unclassified Phylum Proteobacteria, '
'Unclassified Class Gammaproteobacteria, '
'Unclassified Order Enterobacteriales, '
'Unclassified Family Enterbacteriaceae, '
'Genus Escherichia, '
'*Escherichia coli*']
test_list_latex_format = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=False,
color='red')
test_list_latex_comma = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=True,
color='blue')
test_list_raw_format = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=False)
test_list_raw_comma = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=True)
self.assertEqual(test_list_latex_format, known_latex_format[0])
self.assertEqual(test_list_latex_comma, known_latex_comma[0])
self.assertEqual(test_list_raw_format, known_raw_format[0])
self.assertEqual(test_list_raw_comma, known_raw_comma[0])
def test_convert_taxa_to_list(self):
format_list = ['BOLD', 'REG', 'REG', 'REG', 'REG', 'REG', 'COLOR']
known_latex_format = '\\begin{itemize}\n\\item \\textbf{Unclassified'\
' Kingdom Bacteria}\n\\item Unclassified Phylum Proteobacteria\n'\
'\\item Unclassified Class Gammaproteobacteria\n\\item '\
'Unclassified Order Enterobacteriales\n\\item Unclassified Family'\
' Enterbacteriaceae\n\\item Genus \\textit{Escherichia}\n\\'\
'item \\textcolor{red}{\\textit{Escherichia coli}}\n\\end{itemize}'
known_latex_comma = '\\textbf{Unclassified Kingdom Bacteria}, '\
'Unclassified Phylum Proteobacteria, Unclassified Class '\
'Gammaproteobacteria, Unclassified Order Enterobacteriales, '\
'Unclassified Family Enterbacteriaceae, Genus \\textit'\
'{Escherichia}, \\textcolor{blue}{\\textit{Escherichia coli}}'
known_raw_format = '\n o *Unclassified Kingdom Bacteria*\n o '\
'Unclassified Phylum Proteobacteria\n o Unclassified Class '\
'Gammaproteobacteria\n o Unclassified Order Enterobacteriales'\
'\n o Unclassified Family Enterbacteriaceae\n o Genus '\
'Escherichia\n o *Escherichia coli*'
known_raw_comma = '*Unclassified Kingdom Bacteria*, Unclassified '\
'Phylum Proteobacteria, Unclassified Class Gammaproteobacteria, '\
'Unclassified Order Enterobacteriales, Unclassified Family '\
'Enterbacteriaceae, Genus Escherichia, *Escherichia coli*'
test_list_latex_format = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=False,
color='red')
test_list_latex_comma = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=True,
color='blue')
test_list_raw_format = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=False)
test_list_raw_comma = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=True)
self.assertEqual(test_list_latex_format, known_latex_format)
self.assertEqual(test_list_latex_comma, known_latex_comma)
self.assertEqual(test_list_raw_format, known_raw_format)
self.assertEqual(test_list_raw_comma, known_raw_comma)
def main(taxa_table, output_dir, mapping=None, samples_to_analyze=None):
"""Creates LaTeX formatted significant OTU lists
INPUTS:
tax_table -- a numpy array with the relative frequencies of taxonomies
(rows) for each give sample (column)
output_dir -- a directory where the final files should be saved.
mapping -- a 2D dictionary of mapping data where the sample id is keyed
to a dictionary of metadata.
samples_to_analyze -- a list of samples_to_analyze which should be used
to generate data. If None, all the samples will be used.
DEFAULT: None
OUTPUTS:
Generates text files containing LaTex encoded strings which creates a
LaTeX macro dictionary with the information for creating a table of
most abundant taxa, most enriched taxa, and rare and unique taxa. Rare
defined as present in less than 10% of the total population. The unique
taxa are bolded in the lists.
"""
# Sets up the way samples should be converted
SAMPLE_CONVERTER = {'feces': 'fecal',
'oral_cavity': 'oral',
'oral cavity': 'oral',
'skin': 'skin'}
DUMMY = ['', '', '', '']
COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
# Sets table constants
RENDERING = "LATEX"
RARE_THRESH = 0.1
SUM_MIN = 1
FORMAT_SIGNIFIGANCE = ['%1.2f', "%1.2f", "%i", "SKIP"]
SIGNIFIGANCE_HUNDRED = [True, True, False, False]
MACRO_CATS_SIGNIFICANCE = ['enrichTaxon', 'enrichSampl', 'enrichPopul',
'enrichFold']
MACRO_FORM_SIGNIFICANCE = [lambda x: clean_greengenes_string(x,
render_mode='LATEX'),
lambda x: x,
lambda x: x,
lambda x: x]
DUMMY = ['', '', '', '']
COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
FORMAT_ABUNDANCE = ["%1.1f"]
ABUNDANCE_HUNDRED = [True]
MACRO_CATS_ABUNDANCE = ['abundTaxon', 'abundSampl']
MACRO_FORM_ABUNDANCE = [lambda x: clean_greengenes_string(x,
render_mode='LATEX'), lambda x: x]
DATE_FIELD = 'COLLECTION_DATE'
DATE_FORMAT_SHORT = '%m/%d/%y'
DATE_FORMAT_LONG = '%m/%d/%Y'
UNKNOWNS = set(['None', 'NONE', 'none', 'NA', 'na', 'UNKNOWN', 'unknown'])
DATE_OUT = '%B %d, %Y'
TIME_FIELD = 'COLLECTION_TIME'
# Number of taxa shown is an indexing value, it is one less than what is
# actually shown.
NUM_TAXA_SHOW = 5
# Builds the the taxomnomy tree for the table and identifies the
# rare/unique taxa in each sample
tree, all_taxa = build_tree_from_taxontable(taxa_table)
# Sets up samples for which tables are being generated
if samples_to_analyze is not None:
samples_to_test = samples_to_analyze
else:
samples_to_test = all_taxa.keys()
if samples_to_test:
samples_to_test = set(samples_to_test)
tmp = {k: v for k, v in all_taxa.items() if k in samples_to_test}
all_taxa = tmp
if not samples_to_test:
raise ValueError("No samples!")
# Generates lists and tables for each sample
for samp, filtered_table, rare, unique in sample_rare_unique(tree,
tax_table,
all_taxa,
RARE_THRESH):
# Sets up filename
file_name = pjoin(output_dir, 'macros.tex')
def filt_fun(v, i, md):
return v.sum() > 0
filtered_table = filtered_table.filter(filt_fun, axis='observation',
inplace=False)
abund_table = tax_table.filter(filt_fun, axis='observation',
inplace=False)
# Gets sample information for the whole table
abund_sample = abund_table.data(samp)
abund_taxa = abund_table.ids(axis='observation')
# Gets sample information for other filtered samples
filt_taxa = filtered_table.ids(axis='observation')
population = array([filtered_table.data(i, axis='observation') for i in
filtered_table.ids(axis='observation')])
sample_position = filtered_table.index(samp, axis='sample')
filt_sample = filtered_table.data(samp)
population = delete(population, sample_position, 1)
# Converts the lists into greengenes strings for later processing
greengenes_rare = []
greengenes_unique = []
for taxon in rare:
greengenes_rare.append(';'.join(taxon))
for taxon in unique:
greengenes_unique.append(';'.join(taxon))
# Formats the rare and unique lists
rare_format = []
rare_combined = []
for taxon in greengenes_unique:
rare_combined.append(taxon)
rare_format.append('COLOR')
for taxon in greengenes_rare:
rare_combined.append(taxon)
rare_format.append('REG')
number_rare_tax = len(rare_combined)
num_rare = len(rare)
num_unique = len(unique)
rare_formatted = \
convert_taxa_to_list(rare_combined[0:NUM_TAXA_SHOW],
tax_format=rare_format,
render_mode=RENDERING,
comma=True)
if num_unique > 0:
unique_string = ' and \\textcolor{red}{%i unique}' % num_unique
else:
unique_string = ''
if number_rare_tax == 0:
rare_formatted = "There were no rare or unique taxa found in "\
"your sample."
elif 0 < number_rare_tax <= NUM_TAXA_SHOW:
rare_formatted = 'Your sample contained the following rare%s '\
'taxa: %s.' % (unique_string, rare_formatted)
else:
rare_formatted = 'Your sample contained %i rare%s taxa, '\
'including the following: %s.' \
% (num_rare, unique_string,
rare_formatted)
# Calculates abundance rank
(abundance) = calculate_abundance(abund_sample, abund_taxa,
sum_min=SUM_MIN)
# Generates formatted abundance table
formatted_abundance = convert_taxa(abundance[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_ABUNDANCE,
hundredx=ABUNDANCE_HUNDRED)
abundance_formatted = \
build_latex_macro(formatted_abundance,
categories=MACRO_CATS_ABUNDANCE,
format=MACRO_FORM_ABUNDANCE)
(high, low) = calculate_tax_rank_1(sample=filt_sample,
population=population,
taxa=filt_taxa,
critical_value=0.05)
if len(high) == 0:
formatted_high = [['', '', '', '']]*NUM_TAXA_SHOW
elif len(high) < NUM_TAXA_SHOW:
# Formats the known high taxa
formatted_high = \
convert_taxa(high[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_SIGNIFIGANCE,
hundredx=SIGNIFIGANCE_HUNDRED)
# Adds the dummy list to the end
for idx in COUNT:
if idx == (NUM_TAXA_SHOW - len(high)):
break
formatted_high.append(DUMMY)
else:
formatted_high = convert_taxa(high[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_SIGNIFIGANCE,
hundredx=SIGNIFIGANCE_HUNDRED)
high_formatted = build_latex_macro(formatted_high,
categories=MACRO_CATS_SIGNIFICANCE,
format=MACRO_FORM_SIGNIFICANCE)
# Handles date parsing
if mapping is not None and mapping[samp][DATE_FIELD] not in UNKNOWNS:
try:
sample_date = format_date(mapping[samp],
date_field=DATE_FIELD,
d_form_in=DATE_FORMAT_SHORT,
format_out=DATE_OUT)
except:
sample_date = format_date(mapping[samp],
date_field=DATE_FIELD,
d_form_in=DATE_FORMAT_LONG,
format_out=DATE_OUT)
else:
sample_date = 'unknown'
# Removes a zero character from the date
if ',' in sample_date and sample_date[sample_date.index(',')-2] == '0':
zero_pos = sample_date.index(',')-2
sample_date = ''.join([sample_date[:zero_pos],
sample_date[zero_pos+1:]])
else:
sample_date = 'unknown'
# Handles sample parsing
if mapping is not None and mapping[samp][TIME_FIELD] not in UNKNOWNS:
sample_time = mapping[samp][TIME_FIELD].lower()
else:
sample_time = 'unknown'
if mapping is not None:
sample_type_prelim = mapping[samp]['BODY_HABITAT'].split(':')[1]
if sample_type_prelim in SAMPLE_CONVERTER:
sample_type = SAMPLE_CONVERTER[sample_type_prelim]
elif sample_type in UNKNOWNS:
sample_time = 'unknown'
else:
sample_type = sample_type_prelim.lower()
else:
sample_type = 'unknown'
# Saves the file
file_for_editing = open(file_name, 'w')
file_for_editing.write('%% Barcode\n\\def\\barcode{%s}\n\n'
% samp.split('.')[0])
file_for_editing.write('%% Sample Type\n\\def\\sampletype{%s}\n\n'
% sample_type)
file_for_editing.write('%% Sample Date\n\\def\\sampledate{%s}\n'
'\\def\\sampletime{%s}\n\n\n'
% (sample_date, sample_time))
file_for_editing.write('%% Abundance Table\n%s\n\n\n'
% abundance_formatted)
file_for_editing.write('%% Enrichment Table\n%s\n\n\n'
% high_formatted)
file_for_editing.write('%% Rare List\n\\def\\rareList{%s}\n'
% rare_formatted)
file_for_editing.close()
def main(taxa_table, output_dir, samples_to_analyze = None):
"""Creates LaTeX formatted significant OTU lists
INPUTS:
taxa -- a numpy vector with greengenes taxonomy strings
tax_table -- a numpy array with the relative frequencies of taxonomies
(rows) for each give sample (column)
samples_to_analyze -- a numpy vector of sample ids associated with the
tax_table values
output_dir -- a directory where the final files should be saved.
samples_to_analyze -- a list of samples_to_analyze which should be used
to generate data. If this is left empty, all the samples in
the table will be used.
OUTPUTS:
Generates text files containing LaTex encoded strings which creates a
LaTeX macro dictionary with the information for creating a table of most
abundant taxa, most enriched taxa, and rare and unique taxa. Rare
defined as present in less than 10% of the total population. The unique
taxa are bolded in the lists.
"""
# Sets table constants
RARE_THRESHHOLD = 0.1
RENDERING = "LATEX"
FORMAT_SIGNIFIGANCE = ['%1.1f', "%1.2f", "%i", "SKIP"]
SIGNIFIGANCE_HUNDRED = [True, True, False, False]
DUMMY = ['','','','']
COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
FORMAT_ABUNDANCE = ["%1.1f"]
ABUNDANCE_HUNDRED = [True]
MACRO_CATS_SIGNIFICANCE = ['enrichTaxon','enrichSampl', 'enrichPopul',
'enrichFoldd']
MACRO_CATS_ABUNDANCE = ['abundTaxon', 'abundSampl']
FILE_PRECURSER = 'macros_'
FILE_EXTENSION = '.tex'
# Number of taxa shown is an indexing value, it is one less than what is
# actually shown.
NUMBER_OF_TAXA_SHOWN = 5
# Builds the the taxomnomy tree for the table and identifies the
# rare/unique taxa in each sample
tree, all_taxa = build_tree_from_taxontable(taxa_table)
# Sets up samples for which tables are being generated
if not samples_to_analyze == None:
samples_to_test = samples_to_analyze
else:
samples_to_test = all_taxa.keys()
if samples_to_test:
samples_to_test = set(samples_to_test)
tmp = {k:v for k,v in all_taxa.items() if k in samples_to_test}
all_taxa = tmp
if not samples_to_test:
raise ValueError, "No samples!"
# Generates lists and tables for each sample
for samp, filtered_table, rare, unique in sample_rare_unique(tree, \
taxa_table, all_taxa, RARE_THRESHHOLD):
filtered_table = filtered_table.filterObservations(lambda v,i,md:\
v.sum() > 0)
# Gets sample information for other samples
taxa = filtered_table.ObservationIds
population = array([filtered_table.observationData(i) for i in \
filtered_table.ObservationIds])
sample_position = filtered_table.getSampleIndex(samp)
sample = filtered_table.sampleData(samp)
print samp, sum(sample > 0)
population = delete(population, sample_position, 1)
# Converts the lists into greengenes strings for later processing
greengenes_rare = []
greengenes_unique = []
for taxon in rare:
greengenes_rare.append('; '.join(taxon))
for taxon in unique:
greengenes_unique.append('; '.join(taxon))
# Formats the rare and unique lists
rare_format = []
rare_combined = []
for taxon in greengenes_unique:
rare_combined.append(taxon)
rare_format.append('COLOR')
for taxon in greengenes_rare:
rare_combined.append(taxon)
rare_format.append('REG')
number_rare_tax = len(rare_combined)
num_rare = len(rare)
num_unique = len(unique)
rare_formatted = convert_taxa_to_list(rare_combined[0:NUMBER_OF_TAXA_SHOWN],
tax_format = rare_format,
render_mode = RENDERING,
comma = True)
if num_unique > 0:
unique_string = ' and \\textcolor{red}{%i unique}' % num_unique
else:
unique_string = ''
if number_rare_tax == 0:
rare_formatted = "There were no rare or unique taxa found in "\
"your sample."
elif 0 < number_rare_tax <= NUMBER_OF_TAXA_SHOWN:
rare_formatted = 'Your sample contained the following rare %s '\
'taxa: %s.' % (unique_string, rare_formatted)
else:
rare_formatted = 'Your sample contained %i rare and %s taxa, '\
'including the following: %s.' \
% (num_rare, unique_string,
rare_formatted)
# Calculates abundance rank
(abundance) = calculate_abundance(sample, taxa,
abundance_threshhold = 1)
# Generates formatted abundance table
formatted_abundance = convert_taxa(abundance[0:NUMBER_OF_TAXA_SHOWN],
formatting_keys = FORMAT_ABUNDANCE,
hundredx = ABUNDANCE_HUNDRED)
abundance_formatted = generate_latex_macro(formatted_abundance, \
categories = MACRO_CATS_ABUNDANCE)
(high, low) = calculate_tax_rank_1(sample = sample,
population = population,
taxa = taxa,
critical_value = 0.05)
if len(high) < NUMBER_OF_TAXA_SHOWN:
# Formats the known high taxa
formatted_high = convert_taxa(high[0:NUMBER_OF_TAXA_SHOWN],
formatting_keys = FORMAT_SIGNIFIGANCE,
hundredx = SIGNIFIGANCE_HUNDRED)
# Adds the dummy list to the end
for idx in COUNT:
if idx == (NUMBER_OF_TAXA_SHOWN - len(high)):
break
formatted_high.append(DUMMY)
high_formatted = generate_latex_macro(formatted_high, \
categories = MACRO_CATS_SIGNIFICANCE)
else:
formatted_high = convert_taxa(high[0:NUMBER_OF_TAXA_SHOWN],
formatting_keys = FORMAT_SIGNIFIGANCE,
hundredx = SIGNIFIGANCE_HUNDRED)
high_formatted = generate_latex_macro(formatted_high, \
categories = MACRO_CATS_SIGNIFICANCE)
file_name = pjoin(output_dir, '%s%s%s' % (FILE_PRECURSER, samp,
FILE_EXTENSION))
# Saves the file
file_for_editing = open(file_name, 'w')
# file_for_editing.write('% Participant Name\n\\def\\yourname'\
# '{Michael Pollan or longer name}\n\n')
file_for_editing.write('%% Abundance Table\n%s\n\n\n' \
% abundance_formatted)
file_for_editing.write('%% Enrichment Table\n%s\n\n\n' \
% high_formatted)
file_for_editing.write('%% Rare List\n\\def\\rareList{%s}\n' \
% rare_formatted)
file_for_editing.close()
def test_convert_taxa_to_list(self):
format_list = ['BOLD', 'REG', 'REG', 'REG', 'REG', 'REG', 'COLOR']
known_latex_format = [
'\\begin{itemize}\n\\item \\textbf{cont. '
'Unclassified Kingdom Bacteria}\n\\item cont. '
'Unclassified Phylum Proteobacteria\n\\item '
'Unclassified Class Gammaproteobacteria\n\\item'
' Unclassified Order Enterobacteriales\n\\item'
' Unclassified Family Enterbacteriaceae\n\\item'
' Genus \\textit{Escherichia}\n\\item '
'\\textcolor{red}{\\textit{Escherichia coli}}\n'
'\\end{itemize}'
]
known_latex_comma = [
'\\textbf{cont. Unclassified Kingdom Bacteria},'
' cont. Unclassified Phylum Proteobacteria, '
'Unclassified Class Gammaproteobacteria, '
'Unclassified Order Enterobacteriales, '
'Unclassified Family Enterbacteriaceae, '
'Genus \\textit{Escherichia}, \\textcolor{blue}'
'{\\textit{Escherichia coli}}'
]
known_raw_format = [
'\n o *cont. Unclassified Kingdom Bacteria*\n '
'o cont. Unclassified Phylum Proteobacteria\n '
'o Unclassified Class Gammaproteobacteria\n '
'o Unclassified Order Enterobacteriales\n '
'o Unclassified Family Enterbacteriaceae\n '
'o Genus Escherichia\n '
'o *Escherichia coli*'
]
known_raw_comma = [
'*cont. Unclassified Kingdom Bacteria*,'
' cont. Unclassified Phylum Proteobacteria, '
'Unclassified Class Gammaproteobacteria, '
'Unclassified Order Enterobacteriales, '
'Unclassified Family Enterbacteriaceae, '
'Genus Escherichia, '
'*Escherichia coli*'
]
test_list_latex_format = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=False,
color='red')
test_list_latex_comma = convert_taxa_to_list(self.test_list,
render_mode='LATEX',
tax_format=format_list,
comma=True,
color='blue')
test_list_raw_format = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=False)
test_list_raw_comma = convert_taxa_to_list(self.test_list,
render_mode='RAW',
tax_format=format_list,
comma=True)
self.assertEqual(test_list_latex_format, known_latex_format[0])
self.assertEqual(test_list_latex_comma, known_latex_comma[0])
self.assertEqual(test_list_raw_format, known_raw_format[0])
self.assertEqual(test_list_raw_comma, known_raw_comma[0])
