def test_calculate_tax_rank_1(self):
        # Sets up known values
        known_high_10 = [['k__Bacteria; p__Proteobacteria; '\
                          'c__Gammaproteobacteria; o__Enterobacteriales; '\
                          'f__Enterbacteriaceae', 0.002, 7.6e-05, 26.0, 
                          1.4507298345686689e-22], 
                         ['k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852, 
                          7.0, 0.00073396297302392652], 
                         ['k__Bacteria; p__Proteobacteria; '\
                          'c__Gammaproteobacteria; o__Enterobacteriales; '\
                          'f__Enterbacteriaceae; g__Escherichia', 0.1, 0.04714, 
                          2.0, 0.068040408640427208]]
        known_high_05 = [['k__Bacteria; p__Proteobacteria; '\
                          'c__Gammaproteobacteria; o__Enterobacteriales; '\
                          'f__Enterbacteriaceae', 0.002, 7.6e-05, 26.0, 
                          1.4507298345686689e-22], 
                         ['k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852, 
                          7.0, 0.00073396297302392652]]
        known_high_01 = [['k__Bacteria; p__Proteobacteria; '\
                          'c__Gammaproteobacteria; o__Enterobacteriales; '\
                          'f__Enterbacteriaceae', 0.002, 7.6e-05, 
                           26.0, 1.4507298345686689e-22], 
                         ['k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852, 
                          7.0, 0.00073396297302392652]]
        known_low_10  = [['k__Bacteria; p__Actinobacteria; c__Actinobacteria;'\
                          ' o__Actinomycetales; f__Dietziaceae; g__', 
                          0.0, 0.044336, 0.0, 0.016163391728614671], 
                         ['k__Bacteria', 
                         0.02, 0.704584, 0.0, 0.051170083967289066], 
                         ['k__Archaea; p__Crenarchaeota; c__Thaumarchaeota;'\
                         ' o__Cenarchaeales; f__Cenarchaeaceae; '\
                         'g__Nitrosopumilus', 
                         0.03, 0.289032, 0.0, 0.065525264907055]]
        known_low_05  = [['k__Bacteria; p__Actinobacteria; c__Actinobacteria;'\
                          ' o__Actinomycetales; f__Dietziaceae; g__', 
                          0.0, 0.044336, 0.0, 0.016163391728614671]]
        known_low_01  = []
        # Sets up test values
        (test_high_10, test_low_10) = calculate_tax_rank_1(sample = self.sample,
                                                        population = self.pop,
                                                        taxa = self.taxa,
                                                        critical_value = 0.1)

        (test_high_05, test_low_05) = calculate_tax_rank_1(sample = self.sample,
                                                         population = self.pop,
                                                         taxa = self.taxa)
        
        (test_high_01, test_low_01) = calculate_tax_rank_1(sample = self.sample,
                                                        population = self.pop,
                                                        taxa = self.taxa,
                                                        critical_value = 0.01)

        self.assertEqual(known_high_10, test_high_10)
        self.assertEqual(known_low_10, test_low_10)
        self.assertEqual(known_high_05, test_high_05)
        self.assertEqual(known_low_05, test_low_05)
        self.assertEqual(known_high_01, test_high_01)
        self.assertEqual(known_low_01, test_low_01)
Example #2
0
def main(taxa_table, output_dir, mapping=None, samples_to_analyze=None):
    """Creates LaTeX formatted significant OTU lists

    INPUTS:
        tax_table -- a numpy array with the relative frequencies of taxonomies
            (rows) for each give sample (column)

        output_dir -- a directory where the final files should be saved.

        mapping -- a 2D dictionary of mapping data where the sample id is keyed
                    to a dictionary of metadata.

        samples_to_analyze -- a list of samples_to_analyze which should be used
                    to generate data. If None, all the samples will be used.
                    DEFAULT: None

    OUTPUTS:
        Generates text files containing LaTex encoded strings which creates a
        LaTeX macro dictionary with the information for creating a table of
        most abundant taxa, most enriched taxa, and rare and unique taxa. Rare
        defined as present in less than 10% of the total population. The unique
        taxa are bolded in the lists.
    """
    # Sets up the way samples should be converted
    SAMPLE_CONVERTER = {'feces': 'fecal',
                        'oral_cavity': 'oral',
                        'oral cavity': 'oral',
                        'skin': 'skin'}

    DUMMY = ['', '', '', '']
    COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
    # Sets table constants
    RENDERING = "LATEX"
    RARE_THRESH = 0.1

    SUM_MIN = 1

    FORMAT_SIGNIFIGANCE = ['%1.2f', "%1.2f", "%i", "SKIP"]
    SIGNIFIGANCE_HUNDRED = [True, True, False, False]
    MACRO_CATS_SIGNIFICANCE = ['enrichTaxon', 'enrichSampl', 'enrichPopul',
                               'enrichFold']
    MACRO_FORM_SIGNIFICANCE = [lambda x: clean_greengenes_string(x,
                               render_mode='LATEX'),
                               lambda x: x,
                               lambda x: x,
                               lambda x: x]

    DUMMY = ['', '', '', '']
    COUNT = [0, 1, 2, 3, 4, 5, 6, 7]

    FORMAT_ABUNDANCE = ["%1.1f"]
    ABUNDANCE_HUNDRED = [True]
    MACRO_CATS_ABUNDANCE = ['abundTaxon', 'abundSampl']
    MACRO_FORM_ABUNDANCE = [lambda x: clean_greengenes_string(x,
                            render_mode='LATEX'), lambda x: x]

    DATE_FIELD = 'COLLECTION_DATE'
    DATE_FORMAT_SHORT = '%m/%d/%y'
    DATE_FORMAT_LONG = '%m/%d/%Y'

    UNKNOWNS = set(['None', 'NONE', 'none', 'NA', 'na', 'UNKNOWN', 'unknown'])
    DATE_OUT = '%B %d, %Y'
    TIME_FIELD = 'COLLECTION_TIME'

    # Number of taxa shown is an indexing value, it is one less than what is
    # actually shown.
    NUM_TAXA_SHOW = 5

    # Builds the the taxomnomy tree for the table and identifies the
    # rare/unique taxa in each sample
    tree, all_taxa = build_tree_from_taxontable(taxa_table)

    # Sets up samples for which tables are being generated
    if samples_to_analyze is not None:
        samples_to_test = samples_to_analyze
    else:
        samples_to_test = all_taxa.keys()

    if samples_to_test:
        samples_to_test = set(samples_to_test)
        tmp = {k: v for k, v in all_taxa.items() if k in samples_to_test}
        all_taxa = tmp
        if not samples_to_test:
            raise ValueError("No samples!")

    # Generates lists and tables for each sample
    for samp, filtered_table, rare, unique in sample_rare_unique(tree,
                                                                 tax_table,
                                                                 all_taxa,
                                                                 RARE_THRESH):
        # Sets up filename
        file_name = pjoin(output_dir, 'macros.tex')

        def filt_fun(v, i, md):
            return v.sum() > 0

        filtered_table = filtered_table.filter(filt_fun, axis='observation',
                                               inplace=False)
        abund_table = tax_table.filter(filt_fun, axis='observation',
                                       inplace=False)

        # Gets sample information for the whole table
        abund_sample = abund_table.data(samp)
        abund_taxa = abund_table.ids(axis='observation')

        # Gets sample information for other filtered samples
        filt_taxa = filtered_table.ids(axis='observation')
        population = array([filtered_table.data(i, axis='observation') for i in
                            filtered_table.ids(axis='observation')])

        sample_position = filtered_table.index(samp, axis='sample')
        filt_sample = filtered_table.data(samp)

        population = delete(population, sample_position, 1)

        # Converts the lists into greengenes strings for later processing
        greengenes_rare = []
        greengenes_unique = []
        for taxon in rare:
            greengenes_rare.append(';'.join(taxon))
        for taxon in unique:
            greengenes_unique.append(';'.join(taxon))

        # Formats the rare and unique lists
        rare_format = []
        rare_combined = []
        for taxon in greengenes_unique:
            rare_combined.append(taxon)
            rare_format.append('COLOR')
        for taxon in greengenes_rare:
            rare_combined.append(taxon)
            rare_format.append('REG')

        number_rare_tax = len(rare_combined)
        num_rare = len(rare)
        num_unique = len(unique)

        rare_formatted = \
            convert_taxa_to_list(rare_combined[0:NUM_TAXA_SHOW],
                                 tax_format=rare_format,
                                 render_mode=RENDERING,
                                 comma=True)

        if num_unique > 0:
            unique_string = ' and \\textcolor{red}{%i unique}' % num_unique
        else:
            unique_string = ''

        if number_rare_tax == 0:
            rare_formatted = "There were no rare or unique taxa found in "\
                "your sample."

        elif 0 < number_rare_tax <= NUM_TAXA_SHOW:
            rare_formatted = 'Your sample contained the following rare%s '\
                'taxa: %s.' % (unique_string, rare_formatted)

        else:
            rare_formatted = 'Your sample contained %i rare%s taxa, '\
                'including the following: %s.' \
                % (num_rare, unique_string,
                   rare_formatted)

        # Calculates abundance rank
        (abundance) = calculate_abundance(abund_sample, abund_taxa,
                                          sum_min=SUM_MIN)

        # Generates formatted abundance table
        formatted_abundance = convert_taxa(abundance[0:NUM_TAXA_SHOW],
                                           formatting_keys=FORMAT_ABUNDANCE,
                                           hundredx=ABUNDANCE_HUNDRED)

        abundance_formatted = \
            build_latex_macro(formatted_abundance,
                              categories=MACRO_CATS_ABUNDANCE,
                              format=MACRO_FORM_ABUNDANCE)

        (high, low) = calculate_tax_rank_1(sample=filt_sample,
                                           population=population,
                                           taxa=filt_taxa,
                                           critical_value=0.05)

        if len(high) == 0:
            formatted_high = [['', '', '', '']]*NUM_TAXA_SHOW

        elif len(high) < NUM_TAXA_SHOW:
            # Formats the known high taxa
            formatted_high = \
                convert_taxa(high[0:NUM_TAXA_SHOW],
                             formatting_keys=FORMAT_SIGNIFIGANCE,
                             hundredx=SIGNIFIGANCE_HUNDRED)

            # Adds the dummy list to the end
            for idx in COUNT:
                if idx == (NUM_TAXA_SHOW - len(high)):
                    break
                formatted_high.append(DUMMY)

        else:
            formatted_high = convert_taxa(high[0:NUM_TAXA_SHOW],
                                          formatting_keys=FORMAT_SIGNIFIGANCE,
                                          hundredx=SIGNIFIGANCE_HUNDRED)

        high_formatted = build_latex_macro(formatted_high,
                                           categories=MACRO_CATS_SIGNIFICANCE,
                                           format=MACRO_FORM_SIGNIFICANCE)

        # Handles date parsing
        if mapping is not None and mapping[samp][DATE_FIELD] not in UNKNOWNS:
            try:
                sample_date = format_date(mapping[samp],
                                          date_field=DATE_FIELD,
                                          d_form_in=DATE_FORMAT_SHORT,
                                          format_out=DATE_OUT)
            except:
                sample_date = format_date(mapping[samp],
                                          date_field=DATE_FIELD,
                                          d_form_in=DATE_FORMAT_LONG,
                                          format_out=DATE_OUT)
        else:
            sample_date = 'unknown'

        # Removes a zero character from the date
        if ',' in sample_date and sample_date[sample_date.index(',')-2] == '0':
                zero_pos = sample_date.index(',')-2
                sample_date = ''.join([sample_date[:zero_pos],
                                       sample_date[zero_pos+1:]])

        else:
            sample_date = 'unknown'

        # Handles sample parsing
        if mapping is not None and mapping[samp][TIME_FIELD] not in UNKNOWNS:
            sample_time = mapping[samp][TIME_FIELD].lower()
        else:
            sample_time = 'unknown'

        if mapping is not None:
            sample_type_prelim = mapping[samp]['BODY_HABITAT'].split(':')[1]
            if sample_type_prelim in SAMPLE_CONVERTER:
                sample_type = SAMPLE_CONVERTER[sample_type_prelim]
            elif sample_type in UNKNOWNS:
                sample_time = 'unknown'
            else:
                sample_type = sample_type_prelim.lower()
        else:
            sample_type = 'unknown'

        # Saves the file
        file_for_editing = open(file_name, 'w')
        file_for_editing.write('%% Barcode\n\\def\\barcode{%s}\n\n'
                               % samp.split('.')[0])
        file_for_editing.write('%% Sample Type\n\\def\\sampletype{%s}\n\n'
                               % sample_type)
        file_for_editing.write('%% Sample Date\n\\def\\sampledate{%s}\n'
                               '\\def\\sampletime{%s}\n\n\n'
                               % (sample_date, sample_time))
        file_for_editing.write('%% Abundance Table\n%s\n\n\n'
                               % abundance_formatted)
        file_for_editing.write('%% Enrichment Table\n%s\n\n\n'
                               % high_formatted)
        file_for_editing.write('%% Rare List\n\\def\\rareList{%s}\n'
                               % rare_formatted)
        file_for_editing.close()
def main(taxa_table, output_dir, samples_to_analyze = None):

    """Creates LaTeX formatted significant OTU lists

    INPUTS:
        taxa -- a numpy vector with greengenes taxonomy strings

        tax_table -- a numpy array with the relative frequencies of taxonomies
            (rows) for each give sample (column)

        samples_to_analyze -- a numpy vector of sample ids associated with the 
            tax_table values

        output_dir -- a directory where the final files should be saved.

        samples_to_analyze -- a list of samples_to_analyze which should be used 
                    to generate data. If this is left empty, all the samples in 
                    the table will be used.

    OUTPUTS:
        Generates text files containing LaTex encoded strings which creates a 
        LaTeX macro dictionary with the information for creating a table of most
        abundant taxa, most enriched taxa, and rare and unique taxa. Rare 
        defined as present in less than 10% of the total population. The unique 
        taxa are bolded in the lists. 
    """

    # Sets table constants
    RARE_THRESHHOLD = 0.1
    RENDERING = "LATEX"
    FORMAT_SIGNIFIGANCE = ['%1.1f', "%1.2f", "%i", "SKIP"]
    SIGNIFIGANCE_HUNDRED = [True, True, False, False]
    DUMMY  = ['','','','']    
    COUNT = [0, 1, 2, 3, 4, 5, 6, 7]
    FORMAT_ABUNDANCE = ["%1.1f"]
    ABUNDANCE_HUNDRED = [True]
    MACRO_CATS_SIGNIFICANCE = ['enrichTaxon','enrichSampl', 'enrichPopul', 
        'enrichFoldd']
    MACRO_CATS_ABUNDANCE = ['abundTaxon', 'abundSampl']

    FILE_PRECURSER = 'macros_'
    FILE_EXTENSION = '.tex'

    # Number of taxa shown is an indexing value, it is one less than what is 
    # actually shown.
    NUMBER_OF_TAXA_SHOWN = 5

    # Builds the the taxomnomy tree for the table and identifies the 
    # rare/unique taxa in each sample
    tree, all_taxa = build_tree_from_taxontable(taxa_table)

    # Sets up samples for which tables are being generated    
    if not samples_to_analyze == None:
        samples_to_test = samples_to_analyze
    else:
        samples_to_test = all_taxa.keys()

    if samples_to_test:
        samples_to_test = set(samples_to_test)
        tmp = {k:v for k,v in all_taxa.items() if k in samples_to_test}
        all_taxa = tmp
        if not samples_to_test:
            raise ValueError, "No samples!"

    # Generates lists and tables for each sample
    for samp, filtered_table, rare, unique in sample_rare_unique(tree, \
        taxa_table, all_taxa, RARE_THRESHHOLD):
        filtered_table = filtered_table.filterObservations(lambda v,i,md:\
        v.sum() > 0)

        # Gets sample information for other samples
        taxa = filtered_table.ObservationIds       
        population = array([filtered_table.observationData(i) for i in \
            filtered_table.ObservationIds])
        
        sample_position = filtered_table.getSampleIndex(samp)
        sample = filtered_table.sampleData(samp)
        print samp, sum(sample > 0)
                
        population = delete(population, sample_position, 1)
        
        # Converts the lists into greengenes strings for later processing
        greengenes_rare = []
        greengenes_unique = []
        for taxon in rare:
            greengenes_rare.append('; '.join(taxon))
        for taxon in unique:
            greengenes_unique.append('; '.join(taxon))

        # Formats the rare and unique lists          
        rare_format = []
        rare_combined = []
        for taxon in greengenes_unique:
            rare_combined.append(taxon)
            rare_format.append('COLOR')
        for taxon in greengenes_rare:
            rare_combined.append(taxon)
            rare_format.append('REG')

        number_rare_tax = len(rare_combined)
        num_rare = len(rare)
        num_unique = len(unique)

        rare_formatted = convert_taxa_to_list(rare_combined[0:NUMBER_OF_TAXA_SHOWN], 
                                               tax_format = rare_format,
                                               render_mode = RENDERING, 
                                               comma = True)        
     
        if num_unique > 0:
            unique_string = ' and \\textcolor{red}{%i unique}' % num_unique
        else:
            unique_string = ''

        if number_rare_tax == 0:
            rare_formatted = "There were no rare or unique taxa found in "\
                "your sample." 

        elif 0 < number_rare_tax <= NUMBER_OF_TAXA_SHOWN:
            rare_formatted = 'Your sample contained the following rare %s '\
                'taxa: %s.' % (unique_string, rare_formatted)

        else:
            rare_formatted = 'Your sample contained %i rare and %s taxa, '\
                'including the following: %s.' \
                % (num_rare, unique_string, 
                   rare_formatted)


        # Calculates abundance rank
        (abundance) = calculate_abundance(sample, taxa, 
                                          abundance_threshhold = 1)

        # Generates formatted abundance table
        formatted_abundance = convert_taxa(abundance[0:NUMBER_OF_TAXA_SHOWN],                                        
                                        formatting_keys = FORMAT_ABUNDANCE,
                                        hundredx = ABUNDANCE_HUNDRED)

        abundance_formatted = generate_latex_macro(formatted_abundance, \
            categories = MACRO_CATS_ABUNDANCE)

        (high, low) = calculate_tax_rank_1(sample = sample, 
                                           population = population, 
                                           taxa = taxa,
                                           critical_value = 0.05)

        if len(high) < NUMBER_OF_TAXA_SHOWN:
            # Formats the known high taxa
            formatted_high = convert_taxa(high[0:NUMBER_OF_TAXA_SHOWN],
                                          formatting_keys = FORMAT_SIGNIFIGANCE,
                                          hundredx = SIGNIFIGANCE_HUNDRED)
            # Adds the dummy list to the end
            for idx in COUNT:
                if idx == (NUMBER_OF_TAXA_SHOWN - len(high)):
                    break
                formatted_high.append(DUMMY)           

            high_formatted = generate_latex_macro(formatted_high, \
                categories = MACRO_CATS_SIGNIFICANCE)

        else:
            formatted_high = convert_taxa(high[0:NUMBER_OF_TAXA_SHOWN],
                                          formatting_keys = FORMAT_SIGNIFIGANCE,
                                          hundredx = SIGNIFIGANCE_HUNDRED)

            high_formatted = generate_latex_macro(formatted_high, \
                categories = MACRO_CATS_SIGNIFICANCE)


    
        file_name = pjoin(output_dir, '%s%s%s' % (FILE_PRECURSER, samp, 
            FILE_EXTENSION))

        # Saves the file
        file_for_editing = open(file_name, 'w')
        # file_for_editing.write('% Participant Name\n\\def\\yourname'\
        #     '{Michael Pollan or longer name}\n\n')
        file_for_editing.write('%% Abundance Table\n%s\n\n\n' \
            % abundance_formatted)
        file_for_editing.write('%% Enrichment Table\n%s\n\n\n' \
            % high_formatted)
        file_for_editing.write('%% Rare List\n\\def\\rareList{%s}\n' \
            % rare_formatted)
        file_for_editing.close()
Example #4
0
    def test_calculate_tax_rank_1(self):
        # Sets up known values
        known_high_10 = [[
            'k__Bacteria; p__Proteobacteria; '
            'c__Gammaproteobacteria; o__Enterobacteriales; '
            'f__Enterbacteriaceae', 0.002, 7.6e-05, 26.0,
            1.4507298345686689e-22
        ],
                         [
                             'k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852,
                             7.0, 0.00073396297302392771
                         ],
                         [
                             'k__Bacteria; p__Proteobacteria; '
                             'c__Gammaproteobacteria; o__Enterobacteriales; '
                             'f__Enterbacteriaceae; g__Escherichia', 0.1,
                             0.04714, 2.0, 0.068040408640427208
                         ]]
        known_high_05 = [[
            'k__Bacteria; p__Proteobacteria; '
            'c__Gammaproteobacteria; o__Enterobacteriales; '
            'f__Enterbacteriaceae', 0.002, 7.6e-05, 26.0,
            1.4507298345686689e-22
        ],
                         [
                             'k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852,
                             7.0, 0.00073396297302392771
                         ]]
        known_high_01 = [[
            'k__Bacteria; p__Proteobacteria; '
            'c__Gammaproteobacteria; o__Enterobacteriales; '
            'f__Enterbacteriaceae', 0.002, 7.6e-05, 26.0,
            1.4507298345686689e-22
        ],
                         [
                             'k__Bacteria; p__[Proteobacteria]', 0.7, 0.101852,
                             7.0, 0.00073396297302392771
                         ]]
        known_low_10 = [[
            'k__Bacteria; p__Actinobacteria; c__Actinobacteria;'
            ' o__Actinomycetales; f__Dietziaceae; g__', 0.0, 0.044336, 0.0,
            0.016163391728614671
        ], ['k__Bacteria', 0.02, 0.704584, 0.0, 0.051170083967289066],
                        [
                            'k__Archaea; p__Crenarchaeota; c__Thaumarchaeota;'
                            ' o__Cenarchaeales; f__Cenarchaeaceae; '
                            'g__Nitrosopumilus', 0.03, 0.289032, 0.0,
                            0.065525264907055
                        ]]
        known_low_05 = [[
            'k__Bacteria; p__Actinobacteria; c__Actinobacteria;'
            ' o__Actinomycetales; f__Dietziaceae; g__', 0.0, 0.044336, 0.0,
            0.016163391728614671
        ]]
        known_low_01 = []
        # Sets up test values
        (test_high_10, test_low_10) = \
            calculate_tax_rank_1(sample=self.sample,
                                 population=self.pop,
                                 taxa=self.taxa,
                                 critical_value=0.1)

        (test_high_05, test_low_05) = \
            calculate_tax_rank_1(sample=self.sample,
                                 population=self.pop,
                                 taxa=self.taxa)

        (test_high_01, test_low_01) = \
            calculate_tax_rank_1(sample=self.sample,
                                 population=self.pop,
                                 taxa=self.taxa,
                                 critical_value=0.01)

        self.assertEqual(known_high_10, test_high_10)
        self.assertEqual(known_low_10, test_low_10)
        self.assertEqual(known_high_05, test_high_05)
        self.assertEqual(known_low_05, test_low_05)
        self.assertEqual(known_high_01, test_high_01)
        self.assertEqual(known_low_01, test_low_01)