def test_build_latex_macro(self):
"""Tests that a latex macro is generated"""
# Sets up the text functions for formatting
test_funs = [lambda x: clean_greengenes_string(x, render_mode='LATEX'),
lambda x: x,
lambda x: x,
lambda x: x]
# Tests single generation mode
known_macro = '\\def\\Taxonomy{Genus \\textit{Escherichia}}\n'\
'\\def\\Doctor{10.00%}\n'\
'\\def\\Humans{1.00%}\n'\
'\\def\\Fold{10}'
test_macro = build_latex_macro(data=self.test_table[0],
categories=self.header,
format=test_funs)
self.assertEqual(known_macro, test_macro)
# Tests multiple generation mode (a list of lists)
known_macro = '\\def\\TaxonomyA{Genus \\textit{Escherichia}}\n'\
'\\def\\DoctorA{10.00%}\n'\
'\\def\\HumansA{1.00%}\n'\
'\\def\\FoldA{10}\n\n'\
'\\def\\TaxonomyB{\\textit{Escherichia coli}}\n'\
'\\def\\DoctorB{0.10%}\n'\
'\\def\\HumansB{1.00%}\n'\
'\\def\\FoldB{0.10}\n'
test_macro = build_latex_macro(data=self.test_table,
categories=self.header,
format=test_funs)
self.assertEqual(known_macro, test_macro)
def test_build_latex_macro(self):
"""Tests that a latex macro is generated"""
# Sets up the text functions for formatting
test_funs = [
lambda x: clean_greengenes_string(x, render_mode='LATEX'),
lambda x: x, lambda x: x, lambda x: x
]
# Tests single generation mode
known_macro = '\\def\\Taxonomy{Genus \\textit{Escherichia}}\n'\
'\\def\\Doctor{10.00%}\n'\
'\\def\\Humans{1.00%}\n'\
'\\def\\Fold{10}'
test_macro = build_latex_macro(data=self.test_table[0],
categories=self.header,
format=test_funs)
self.assertEqual(known_macro, test_macro)
# Tests multiple generation mode (a list of lists)
known_macro = '\\def\\TaxonomyA{Genus \\textit{Escherichia}}\n'\
'\\def\\DoctorA{10.00%}\n'\
'\\def\\HumansA{1.00%}\n'\
'\\def\\FoldA{10}\n\n'\
'\\def\\TaxonomyB{\\textit{Escherichia coli}}\n'\
'\\def\\DoctorB{0.10%}\n'\
'\\def\\HumansB{1.00%}\n'\
'\\def\\FoldB{0.10}\n'
test_macro = build_latex_macro(data=self.test_table,
categories=self.header,
format=test_funs)
self.assertEqual(known_macro, test_macro)
def test_clean_otu_string(self):
known_raw = [
'*cont. Kingdom Bacteria*', 'cont. Phylum Proteobacteria',
'Class Gammaproteobacteria', 'Order Enterobacteriales',
'Family Enterbacteriaceae', '*Genus Escherichia*',
'*Escherichia coli*'
]
known_latex = [
'\\textbf{cont. Unclassified Kingdom Bacteria}',
'cont. Unclassified Phylum Proteobacteria',
'Unclassified Class Gammaproteobacteria',
'Unclassified Order Enterobacteriales',
'Unclassified Family Enterbacteriaceae',
'\\textcolor{red}{Genus \\textit{Escherichia}}',
'\\textcolor{blue}{\\textit{Escherichia coli}}'
]
for idx, taxon in enumerate(self.test_list):
# Sets the format
if idx == 0:
form = 'BOLD'
color = 'red'
elif idx == 5:
form = 'COLOR'
color = 'red'
elif idx == 6:
form = 'COLOR'
color = 'blue'
else:
form = 'NONE'
color = 'red'
# Generates the cleaned string
test_string_raw = clean_greengenes_string(taxon,
render_mode='RAW',
format=form,
color=color)
test_string_latex = clean_greengenes_string(taxon,
render_mode='LATEX',
format=form,
color=color,
unclassified=True)
self.assertEqual(test_string_raw, known_raw[idx])
self.assertEqual(test_string_latex, known_latex[idx])
def test_clean_otu_string(self):
known_raw = ['*cont. Kingdom Bacteria*',
'cont. Phylum Proteobacteria',
'Class Gammaproteobacteria',
'Order Enterobacteriales',
'Family Enterbacteriaceae',
'*Genus Escherichia*',
'*Escherichia coli*']
known_latex = ['\\textbf{cont. Unclassified Kingdom Bacteria}',
'cont. Unclassified Phylum Proteobacteria',
'Unclassified Class Gammaproteobacteria',
'Unclassified Order Enterobacteriales',
'Unclassified Family Enterbacteriaceae',
'\\textcolor{red}{Genus \\textit{Escherichia}}',
'\\textcolor{blue}{\\textit{Escherichia coli}}']
for idx, taxon in enumerate(self.test_list):
# Sets the format
if idx == 0:
form = 'BOLD'
color = 'red'
elif idx == 5:
form = 'COLOR'
color = 'red'
elif idx == 6:
form = 'COLOR'
color = 'blue'
else:
form = 'NONE'
color = 'red'
# Generates the cleaned string
test_string_raw = clean_greengenes_string(taxon,
render_mode='RAW',
format=form,
color=color)
test_string_latex = clean_greengenes_string(taxon,
render_mode='LATEX',
format=form,
color=color,
unclassified=True)
self.assertEqual(test_string_raw, known_raw[idx])
self.assertEqual(test_string_latex, known_latex[idx])
def main(tax_table, output_dir, samples_to_analyze=None):
"""Generates pie chart of the most abundant twelve taxa in the sample
INPUTS:
otu_table -- a biom formatted taxonomy table at the desired level of
resolution
output_dir -- the location of the directory where output files should
be stored.
samples_to_analyze -- a list of sample ids to plot. If no value is
passed, then all samples in the biom table are analyzed.
OUTPUTS:
A pdf of the piechart summarizing the most abundant taxa will be
generated and saved to the output directory. These will follow the
naming convention PIECHART_<SAMPLEID>.pdf.
"""
# Creates the text around hte file name
FILENAME_BEFORE = 'piechart_'
FILENAME_AFTER = '.pdf'
# Handles string cleaning
RENDER = 'LATEX'
UNCLASSIFIED = False
# Sets up the rare threshhold for
RARE_THRESH = 0.0
SUM_MIN = 1
# Sets up axis parameters
AXIS_LENGTH = 7.25
AXIS_BORDER = 0.01
AXIS_TITLE = 0
AXIS_LEGEND = 7
# Modifies the axis limits
AX_LIMS = [-1.05, 1.05]
# Sets up constants for getting the colormap and plotting
MAP_NAME = 'BrBG'
NUM_SHOW = 12
OTHER_COLOR = array([[85/255, 85/255, 85/255]])
# Sets up plotting parameters
FIG_LEGEND = True
FIG_COLOR_EDGE = False
FIG_LEG_FRAME = False
FIG_LEG_OFFSET = [0.95, 0.025, 1.0, 0.95]
# Sets up the the legend font
LEG_FONT = FontProperties()
LEG_FONT.set_size(28)
LEG_FONT.set_family('sans-serif')
# Sets the general font properties
use_latex = True
rc_font_family = 'sans-serif'
rc_font = ['Helvetica', 'Arial']
# Sets up the colormap
colormap = translate_colors((NUM_SHOW-1), MAP_NAME)
colormap = vstack((colormap, OTHER_COLOR))
# Sets up plotting constants
(axis_dims, fig_dims) = calculate_dimensions_rectangle(
axis_width=AXIS_LENGTH, axis_height=AXIS_LENGTH, border=AXIS_BORDER,
title=AXIS_TITLE, legend=AXIS_LEGEND)
# Walks over a taxa tree and prioritizes based on taxonomy
(tree, all_taxa) = build_tree_from_taxontable(tax_table)
# Sets up samples for which tables are being generated
if samples_to_analyze is not None:
samples_to_test = samples_to_analyze
else:
samples_to_test = all_taxa.keys()
# Checks the samples exist
if samples_to_test:
samples_to_test = set(samples_to_test)
tmp = {k: v for k, v in all_taxa.items() if k in samples_to_test}
all_taxa = tmp
if not samples_to_test:
raise ValueError("No samples!")
# Walks over the table
filt_fun = lambda v, i, md: v.sum() > 0
for samp, filtered_table, rare, unique in sample_rare_unique(tree,
tax_table,
all_taxa,
RARE_THRESH):
# abund_fun = lambda v, i, md: i in all_taxa[samp]
filtered_table = tax_table.filterObservations(filt_fun)
sample_data = filtered_table.sampleData(samp)
taxa = filtered_table.ObservationIds
# Calculates abundance and limits to the top n samples.
abund_rank = calculate_abundance(sample=sample_data,
taxa=taxa,
sum_min=SUM_MIN)
abund_rank = abund_rank[:(NUM_SHOW-1)]
# Cleans the greengenes strings and adds an "Other" Category for
# missing taxa
[sample_tax, sample_freq] = [list(a) for a in zip(*abund_rank)]
clean_tax = [clean_greengenes_string(tax, RENDER,
unclassified=UNCLASSIFIED)
for tax in sample_tax]
clean_tax.append('Other')
sample_freq.append(1-sum(sample_freq))
# Sets up the sample filename
filename = pjoin(output_dir, '%s%s%s' % (FILENAME_BEFORE, samp,
FILENAME_AFTER))
# Creates the pie chart
render_single_pie(data_vec=sample_freq,
group_names=clean_tax,
axis_dims=axis_dims,
fig_dims=fig_dims,
file_out=filename,
legend=FIG_LEGEND,
colors=colormap,
show_edge=FIG_COLOR_EDGE,
legend_frame=FIG_LEG_FRAME,
rc_font=rc_font,
legend_offset=FIG_LEG_OFFSET,
rc_fam=rc_font_family,
legend_font=LEG_FONT,
use_latex=use_latex,
x_lims=AX_LIMS,
y_lims=AX_LIMS)
def main(tax_table, output_dir, samples_to_analyze=None):
"""Generates pie chart of the most abundant twelve taxa in the sample
INPUTS:
otu_table -- a biom formatted taxonomy table at the desired level of
resolution
output_dir -- the location of the directory where output files should
be stored.
samples_to_analyze -- a list of sample ids to plot. If no value is
passed, then all samples in the biom table are analyzed.
OUTPUTS:
A pdf of the piechart summarizing the most abundant taxa will be
generated and saved to the output directory. These will follow the
naming convention PIECHART_<SAMPLEID>.pdf.
"""
# Creates the text around hte file name
FILENAME_BEFORE = 'piechart_'
FILENAME_AFTER = '.pdf'
# Handles string cleaning
RENDER = 'LATEX'
UNCLASSIFIED = False
# Sets up the rare threshhold for
RARE_THRESH = 0.0
SUM_MIN = 1
# Sets up axis parameters
AXIS_LENGTH = 7.25
AXIS_BORDER = 0.01
AXIS_TITLE = 0
AXIS_LEGEND = 7
# Modifies the axis limits
AX_LIMS = [-1.05, 1.05]
# Sets up constants for getting the colormap and plotting
MAP_NAME = 'BrBG'
NUM_SHOW = 12
OTHER_COLOR = array([[85 / 255, 85 / 255, 85 / 255]])
# Sets up plotting parameters
FIG_LEGEND = True
FIG_COLOR_EDGE = False
FIG_LEG_FRAME = False
FIG_LEG_OFFSET = [0.95, 0.025, 1.0, 0.95]
# Sets up the the legend font
LEG_FONT = FontProperties()
LEG_FONT.set_size(28)
LEG_FONT.set_family('sans-serif')
# Sets the general font properties
use_latex = True
rc_font_family = 'sans-serif'
rc_font = ['Helvetica', 'Arial']
# Sets up the colormap
colormap = translate_colors((NUM_SHOW - 1), MAP_NAME)
colormap = vstack((colormap, OTHER_COLOR))
# Sets up plotting constants
(axis_dims,
fig_dims) = calculate_dimensions_rectangle(axis_width=AXIS_LENGTH,
axis_height=AXIS_LENGTH,
border=AXIS_BORDER,
title=AXIS_TITLE,
legend=AXIS_LEGEND)
# Walks over a taxa tree and prioritizes based on taxonomy
(tree, all_taxa) = build_tree_from_taxontable(tax_table)
# Sets up samples for which tables are being generated
if samples_to_analyze is not None:
samples_to_test = samples_to_analyze
else:
samples_to_test = all_taxa.keys()
# Checks the samples exist
if samples_to_test:
samples_to_test = set(samples_to_test)
tmp = {k: v for k, v in all_taxa.items() if k in samples_to_test}
all_taxa = tmp
if not samples_to_test:
raise ValueError("No samples!")
# Walks over the table
filt_fun = lambda v, i, md: v.sum() > 0
for samp, filtered_table, rare, unique in sample_rare_unique(
tree, tax_table, all_taxa, RARE_THRESH):
# abund_fun = lambda v, i, md: i in all_taxa[samp]
filtered_table = tax_table.filterObservations(filt_fun)
sample_data = filtered_table.sampleData(samp)
taxa = filtered_table.ObservationIds
# Calculates abundance and limits to the top n samples.
abund_rank = calculate_abundance(sample=sample_data,
taxa=taxa,
sum_min=SUM_MIN)
abund_rank = abund_rank[:(NUM_SHOW - 1)]
# Cleans the greengenes strings and adds an "Other" Category for
# missing taxa
[sample_tax, sample_freq] = [list(a) for a in zip(*abund_rank)]
clean_tax = [
clean_greengenes_string(tax, RENDER, unclassified=UNCLASSIFIED)
for tax in sample_tax
]
clean_tax.append('Other')
sample_freq.append(1 - sum(sample_freq))
# Sets up the sample filename
filename = pjoin(output_dir,
'%s%s%s' % (FILENAME_BEFORE, samp, FILENAME_AFTER))
# Creates the pie chart
render_single_pie(data_vec=sample_freq,
group_names=clean_tax,
axis_dims=axis_dims,
fig_dims=fig_dims,
file_out=filename,
legend=FIG_LEGEND,
colors=colormap,
show_edge=FIG_COLOR_EDGE,
legend_frame=FIG_LEG_FRAME,
rc_font=rc_font,
legend_offset=FIG_LEG_OFFSET,
rc_fam=rc_font_family,
legend_font=LEG_FONT,
use_latex=use_latex,
x_lims=AX_LIMS,
y_lims=AX_LIMS)
def main(taxa_table, output_dir, mapping=None, samples_to_analyze=None):
"""Creates LaTeX formatted significant OTU lists
INPUTS:
tax_table -- a numpy array with the relative frequencies of taxonomies
(rows) for each give sample (column)
output_dir -- a directory where the final files should be saved.
mapping -- a 2D dictionary of mapping data where the sample id is keyed
to a dictionary of metadata.
samples_to_analyze -- a list of samples_to_analyze which should be used
to generate data. If None, all the samples will be used.
DEFAULT: None
OUTPUTS:
Generates text files containing LaTex encoded strings which creates a
LaTeX macro dictionary with the information for creating a table of
most abundant taxa, most enriched taxa, and rare and unique taxa. Rare
defined as present in less than 10% of the total population. The unique
taxa are bolded in the lists.
"""
# Sets up the way samples should be converted
SAMPLE_CONVERTER = {'feces': 'fecal',
'oral_cavity': 'oral',
'oral cavity': 'oral',
'skin': 'skin'}
DUMMY = ['', '', '', '']
COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
# Sets table constants
RENDERING = "LATEX"
RARE_THRESH = 0.1
SUM_MIN = 1
FORMAT_SIGNIFIGANCE = ['%1.2f', "%1.2f", "%i", "SKIP"]
SIGNIFIGANCE_HUNDRED = [True, True, False, False]
MACRO_CATS_SIGNIFICANCE = ['enrichTaxon', 'enrichSampl', 'enrichPopul',
'enrichFold']
MACRO_FORM_SIGNIFICANCE = [lambda x: clean_greengenes_string(x,
render_mode='LATEX'),
lambda x: x,
lambda x: x,
lambda x: x]
DUMMY = ['', '', '', '']
COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
FORMAT_ABUNDANCE = ["%1.1f"]
ABUNDANCE_HUNDRED = [True]
MACRO_CATS_ABUNDANCE = ['abundTaxon', 'abundSampl']
MACRO_FORM_ABUNDANCE = [lambda x: clean_greengenes_string(x,
render_mode='LATEX'), lambda x: x]
DATE_FIELD = 'COLLECTION_DATE'
DATE_FORMAT_SHORT = '%m/%d/%y'
DATE_FORMAT_LONG = '%m/%d/%Y'
UNKNOWNS = set(['None', 'NONE', 'none', 'NA', 'na', 'UNKNOWN', 'unknown'])
DATE_OUT = '%B %d, %Y'
TIME_FIELD = 'COLLECTION_TIME'
# Number of taxa shown is an indexing value, it is one less than what is
# actually shown.
NUM_TAXA_SHOW = 5
# Builds the the taxomnomy tree for the table and identifies the
# rare/unique taxa in each sample
tree, all_taxa = build_tree_from_taxontable(taxa_table)
# Sets up samples for which tables are being generated
if samples_to_analyze is not None:
samples_to_test = samples_to_analyze
else:
samples_to_test = all_taxa.keys()
if samples_to_test:
samples_to_test = set(samples_to_test)
tmp = {k: v for k, v in all_taxa.items() if k in samples_to_test}
all_taxa = tmp
if not samples_to_test:
raise ValueError("No samples!")
# Generates lists and tables for each sample
for samp, filtered_table, rare, unique in sample_rare_unique(tree,
tax_table,
all_taxa,
RARE_THRESH):
# Sets up filename
file_name = pjoin(output_dir, 'macros.tex')
def filt_fun(v, i, md):
return v.sum() > 0
filtered_table = filtered_table.filter(filt_fun, axis='observation',
inplace=False)
abund_table = tax_table.filter(filt_fun, axis='observation',
inplace=False)
# Gets sample information for the whole table
abund_sample = abund_table.data(samp)
abund_taxa = abund_table.ids(axis='observation')
# Gets sample information for other filtered samples
filt_taxa = filtered_table.ids(axis='observation')
population = array([filtered_table.data(i, axis='observation') for i in
filtered_table.ids(axis='observation')])
sample_position = filtered_table.index(samp, axis='sample')
filt_sample = filtered_table.data(samp)
population = delete(population, sample_position, 1)
# Converts the lists into greengenes strings for later processing
greengenes_rare = []
greengenes_unique = []
for taxon in rare:
greengenes_rare.append(';'.join(taxon))
for taxon in unique:
greengenes_unique.append(';'.join(taxon))
# Formats the rare and unique lists
rare_format = []
rare_combined = []
for taxon in greengenes_unique:
rare_combined.append(taxon)
rare_format.append('COLOR')
for taxon in greengenes_rare:
rare_combined.append(taxon)
rare_format.append('REG')
number_rare_tax = len(rare_combined)
num_rare = len(rare)
num_unique = len(unique)
rare_formatted = \
convert_taxa_to_list(rare_combined[0:NUM_TAXA_SHOW],
tax_format=rare_format,
render_mode=RENDERING,
comma=True)
if num_unique > 0:
unique_string = ' and \\textcolor{red}{%i unique}' % num_unique
else:
unique_string = ''
if number_rare_tax == 0:
rare_formatted = "There were no rare or unique taxa found in "\
"your sample."
elif 0 < number_rare_tax <= NUM_TAXA_SHOW:
rare_formatted = 'Your sample contained the following rare%s '\
'taxa: %s.' % (unique_string, rare_formatted)
else:
rare_formatted = 'Your sample contained %i rare%s taxa, '\
'including the following: %s.' \
% (num_rare, unique_string,
rare_formatted)
# Calculates abundance rank
(abundance) = calculate_abundance(abund_sample, abund_taxa,
sum_min=SUM_MIN)
# Generates formatted abundance table
formatted_abundance = convert_taxa(abundance[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_ABUNDANCE,
hundredx=ABUNDANCE_HUNDRED)
abundance_formatted = \
build_latex_macro(formatted_abundance,
categories=MACRO_CATS_ABUNDANCE,
format=MACRO_FORM_ABUNDANCE)
(high, low) = calculate_tax_rank_1(sample=filt_sample,
population=population,
taxa=filt_taxa,
critical_value=0.05)
if len(high) == 0:
formatted_high = [['', '', '', '']]*NUM_TAXA_SHOW
elif len(high) < NUM_TAXA_SHOW:
# Formats the known high taxa
formatted_high = \
convert_taxa(high[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_SIGNIFIGANCE,
hundredx=SIGNIFIGANCE_HUNDRED)
# Adds the dummy list to the end
for idx in COUNT:
if idx == (NUM_TAXA_SHOW - len(high)):
break
formatted_high.append(DUMMY)
else:
formatted_high = convert_taxa(high[0:NUM_TAXA_SHOW],
formatting_keys=FORMAT_SIGNIFIGANCE,
hundredx=SIGNIFIGANCE_HUNDRED)
high_formatted = build_latex_macro(formatted_high,
categories=MACRO_CATS_SIGNIFICANCE,
format=MACRO_FORM_SIGNIFICANCE)
# Handles date parsing
if mapping is not None and mapping[samp][DATE_FIELD] not in UNKNOWNS:
try:
sample_date = format_date(mapping[samp],
date_field=DATE_FIELD,
d_form_in=DATE_FORMAT_SHORT,
format_out=DATE_OUT)
except:
sample_date = format_date(mapping[samp],
date_field=DATE_FIELD,
d_form_in=DATE_FORMAT_LONG,
format_out=DATE_OUT)
else:
sample_date = 'unknown'
# Removes a zero character from the date
if ',' in sample_date and sample_date[sample_date.index(',')-2] == '0':
zero_pos = sample_date.index(',')-2
sample_date = ''.join([sample_date[:zero_pos],
sample_date[zero_pos+1:]])
else:
sample_date = 'unknown'
# Handles sample parsing
if mapping is not None and mapping[samp][TIME_FIELD] not in UNKNOWNS:
sample_time = mapping[samp][TIME_FIELD].lower()
else:
sample_time = 'unknown'
if mapping is not None:
sample_type_prelim = mapping[samp]['BODY_HABITAT'].split(':')[1]
if sample_type_prelim in SAMPLE_CONVERTER:
sample_type = SAMPLE_CONVERTER[sample_type_prelim]
elif sample_type in UNKNOWNS:
sample_time = 'unknown'
else:
sample_type = sample_type_prelim.lower()
else:
sample_type = 'unknown'
# Saves the file
file_for_editing = open(file_name, 'w')
file_for_editing.write('%% Barcode\n\\def\\barcode{%s}\n\n'
% samp.split('.')[0])
file_for_editing.write('%% Sample Type\n\\def\\sampletype{%s}\n\n'
% sample_type)
file_for_editing.write('%% Sample Date\n\\def\\sampledate{%s}\n'
'\\def\\sampletime{%s}\n\n\n'
% (sample_date, sample_time))
file_for_editing.write('%% Abundance Table\n%s\n\n\n'
% abundance_formatted)
file_for_editing.write('%% Enrichment Table\n%s\n\n\n'
% high_formatted)
file_for_editing.write('%% Rare List\n\\def\\rareList{%s}\n'
% rare_formatted)
file_for_editing.close()