Example #1
0
 def __init__(self,log_fp=None,params=None,qiime_config=None,open_mode='w'):
     if log_fp:
         self._f = open(log_fp,open_mode)
     else:
         self._f = None
     start_time = datetime.now().strftime('%H:%M:%S on %d %b %Y')
     self.write('Logging started at %s\n' % start_time)
     self.write('QIIME version: %s\n\n' % get_qiime_library_version())
     self.writeQiimeConfig(qiime_config)
     self.writeParams(params)
Example #2
0
def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    qiime_config = load_qiime_config()
    
    rdp_jarpath = get_rdp_jarpath()
    if rdp_jarpath == None:
        rdp_version = "Not installed."
    else:
        rdp_version = split(rdp_jarpath)[1]

    java_version = get_java_version()
    if java_version is None:
        java_version = "Not installed."

    system_info = [
     ("Platform", platform),
     ("Python version",python_version.replace('\n', ' ')),
     ("Python executable",executable)]
    max_len =  max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print  "==================" 
    for v in system_info:
        print "%*s:\t%s" % (max_len,v[0],v[1])

    version_info = [
     ("PyCogent version", pycogent_lib_version),
     ("NumPy version", numpy_lib_version),
     ("matplotlib version", matplotlib_lib_version),
     ("biom-format version", biom_lib_version),
     ("qcli version", qcli_lib_version),
     ("QIIME library version", get_qiime_library_version()),
     ("QIIME script version", __version__),
     ("PyNAST version (if installed)", pynast_lib_version),
     ("RDP Classifier version (if installed)", rdp_version),
     ("Java version (if installed)", java_version),
     ("Emperor version", emperor_lib_version)]

    max_len =  max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print  "===================" 
    for v in version_info:
        print "%*s:\t%s" % (max_len,v[0],v[1])
    
    print "\nQIIME config values"
    print  "==================="    
    max_len =  max([len(key) for key in qiime_config])
    for key,value in  qiime_config.items():
        print "%*s:\t%s"%(max_len,key,value)

    #run the Testcase.main function to do the tests
    # need to mess with the arg string, otherwise TestCase complains
    if (opts.test):
        print "\n\nrunning checks:\n"
        test_main(argv=["","-v"])
Example #3
0
def format_emperor_autograph(metadata_fp, coords_fp, language='HTML'):
    """Create a signature with some meta-data of the Emperor package

    language: language to which it will be formatted as a multi-line comment

    """

    # supported open and closing of multi-line comments for different languages
    _languages = {
        'HTML': ('<!--', '-->'),
        'Python': ('"""', '"""'),
        'C': ('/*', '*/'),
        'Bash': ('<<COMMENT', 'COMMENT')
    }

    assert language in _languages.keys(), '%s is not a supported language' %\
        language

    autograph = []
    autograph.append(_languages[language][0])
    autograph.append("*Summary of Emperor's Information*")

    # add the day and time at which the command was called
    autograph.append(datetime.now().strftime('Command executed on %B %d, %Y at'
                                             ' %H:%M:%S'))

    # add library version and SHA-1 if available
    autograph.append('Emperor Version: %s' % get_emperor_library_version())
    autograph.append('QIIME Version: %s' % get_qiime_library_version())
    autograph.append('HostName: %s' % gethostname())

    # full path to input files
    autograph.append('Metadata: %s' % abspath(metadata_fp))
    autograph.append('Coordinates: %s' % abspath(coords_fp))

    if any([True for element in argv if 'make_emperor.py' in element]):
        autograph.append('Command: %s' % ' '.join(argv))
    else:
        autograph.append('Command: Cannot find direct call to make_emperor.py')
    autograph.append(_languages[language][1])

    return '%s' % '\n'.join(autograph)
Example #4
0
def format_emperor_autograph(metadata_fp, coords_fp, language='HTML'):
    """Create a signature with some meta-data of the Emperor package

    language: language to which it will be formatted as a multi-line comment

    """

    # supported open and closing of multi-line comments for different languages
    _languages = {'HTML':('<!--', '-->'), 'Python':('"""', '"""'), 'C':('/*',
        '*/'), 'Bash':('<<COMMENT', 'COMMENT')}

    assert language in _languages.keys(), '%s is not a supported language' %\
        language

    autograph = []
    autograph.append(_languages[language][0])
    autograph.append("*Summary of Emperor's Information*")

    # add the day and time at which the command was called
    autograph.append(datetime.now().strftime('Command executed on %B %d, %Y at'
        ' %H:%M:%S'))

    # add library version and SHA-1 if available
    autograph.append('Emperor Version: %s' %  get_emperor_library_version())
    autograph.append('QIIME Version: %s' % get_qiime_library_version())
    autograph.append('HostName: %s' % gethostname())

    # full path to input files
    autograph.append('Metadata: %s' % abspath(metadata_fp))
    autograph.append('Coordinates: %s' % abspath(coords_fp))

    if any([True for element in argv if 'make_emperor.py' in element]):
        autograph.append('Command: %s' % ' '.join(argv))
    else:
        autograph.append('Command: Cannot find direct call to make_emperor.py')
    autograph.append(_languages[language][1])

    return '%s' % '\n'.join(autograph)
Example #5
0
def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    if opts.haiku:
        print "QIIME provides insight\nmicrobial in nature\nto ecology"
        exit(0)

    qiime_config = load_qiime_config()
    test = opts.test
    qiime_base_install = opts.qiime_base_install

    rdp_jarpath = get_rdp_jarpath()
    if rdp_jarpath is None:
        rdp_version = "Not installed."
    else:
        rdp_version = split(rdp_jarpath)[1]

    java_version = get_java_version()
    if java_version is None:
        java_version = "Not installed."

    system_info = [
        ("Platform", platform),
        ("Python version", python_version.replace('\n', ' ')),
        ("Python executable", executable)]
    max_len = max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print "=================="
    for v in system_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    version_info = [
        ("PyCogent version", pycogent_lib_version),
        ("NumPy version", numpy_lib_version),
        ("SciPy version", scipy_lib_version),
        ("matplotlib version", matplotlib_lib_version),
        ("biom-format version", biom_lib_version),
        ("qcli version", qcli_lib_version),
        ("pyqi version", pyqi_lib_version),
        ("scikit-bio version", skbio_lib_version),
        ("QIIME library version", get_qiime_library_version()),
        ("QIIME script version", __version__),
        ("PyNAST version (if installed)", pynast_lib_version),
        ("Emperor version", emperor_lib_version)]
    if not qiime_base_install:
        version_info += [
            ("RDP Classifier version (if installed)", rdp_version),
            ("Java version (if installed)", java_version)]

    max_len = max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print "==================="
    for v in version_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME config values"
    print "==================="
    max_len = max([len(key) for key in qiime_config])
    for key, value in qiime_config.items():
        print "%*s:\t%s" % (max_len, key, value)

    if test:
        if qiime_base_install:
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyBase)
        else:
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyFull)
        if opts.verbose:
            verbosity = 2
        else:
            verbosity = 1
        TextTestRunner(stream=stdout, verbosity=verbosity).run(suite)
Example #6
0
 def test_format_biom_table(self):
     """ Formatting of BIOM table correctly includes "generated-by" information 
     """
     generated_by = "QIIME " + get_qiime_library_version()
     self.assertTrue(generated_by in format_biom_table(self.biom1))
Example #7
0
def format_biom_table(biom_table):
    """ Given a biom-format Table object, returns that Table as a BIOM string"""
    generated_by_str = "QIIME " + get_qiime_library_version()
    return biom_table.getBiomFormatJsonString(generated_by_str)
def write_otu_table(data_access, samples_list, tax_name, file_name_prefix,
                    tmp_prefix, otu_table_file_dir, otu_table_file_dir_db):
    """ Write out the OTU-table """
    
    # Get the OTU-Ids and counts for each sample (OTU-Table)
    sample_counts={}
    otus=[]
    updated_sample_list=[]
    
    # iterate over samples and get the counts
    for i,sample_name1 in enumerate(samples_list):
       sample_counts[sample_name1]={}
       user_data=data_access.getOTUTable(True, sample_name1, 'UCLUST_REF', 97,
                                         'GREENGENES_REFERENCE',97)
       for row in user_data:
           if row[0] not in otus:
               otus.append(str(row[0]))
           if sample_counts[sample_name1].has_key(str(row[0])):
               raise ValueError, 'Duplicate prokmsa ids! - %s ' % sample_name1
           
           ## store OTU counts
           sample_counts[sample_name1][str(row[0])]=row[1]

       # create a list of only samples with OTU hits
       if sample_counts[sample_name1]!={}:
           updated_sample_list.append(sample_name1)
    
    # flush some memory
    samples_list=[]

    # prepare data for json dictionary
    dict_for_json={}
    dict_for_json['id']=None
    dict_for_json['format']=get_biom_format_version_string()
    dict_for_json['format_url']=get_biom_format_url_string()
    dict_for_json['type']="OTU table"
    dict_for_json['generated_by']="QIIME-DB %s" % get_qiime_library_version()
    dict_for_json['date']= "%s" % datetime.now().isoformat()
    dict_for_json['matrix_type']="sparse"
    dict_for_json['matrix_element_type']="int"
    dict_for_json['shape']=[len(otus),len(updated_sample_list)]
    dict_for_json['rows']=[]
    dict_for_json["data"]=[]
    dict_for_json["columns"]=[]

    # create the sparse data output
    for i,sample in enumerate(updated_sample_list):
        dict_for_json["columns"].append({"id":sample, "metadata":None})
        for j, otu in enumerate(otus):
            if sample_counts.has_key(sample):
                if sample_counts[sample].has_key(otu):
                    dict_for_json["data"].append([j, i, 
                                                  sample_counts[sample][otu]])

    # Get Taxonomy information for each observation
    taxonomy=[]
    for i in otus:
       tax_str=data_access.getGGTaxonomy(True,i,tax_name+'_tax_string')
       if not tax_str:
           dict_for_json['rows'].append({"id":"GG_OTU_1", "metadata":None})
       else:
           dict_for_json['rows'].append({"id":i, 
                                    "metadata":{'taxonomy':tax_str.split(';')}})


    json_dump= dumps(dict_for_json)
    
    # flush dict
    dict_for_json={}
    
    """
    # we are dumping json now instead 
    # create the biom object
    otu_biom=table_factory(otu_table,samples_list,otus,
                          sample_metadata=None,
                          observation_metadata=taxonomy,
                          table_id=None,
                          constructor=SparseOTUTable,
                          dtype=int)
    """
    # create OTU table filepaths
    otu_table_fname = file_name_prefix+'_'+tmp_prefix+'_otu_table.biom'
    otu_table_filepath=os.path.join(otu_table_file_dir, otu_table_fname)
    otu_table_filepath_db=os.path.join(otu_table_file_dir_db, otu_table_fname)
    
    # write OTU table
    otu_table_write=open(otu_table_filepath,'w')
    otu_table_write.write(json_dump)
    otu_table_write.close()
    
    return otu_table_filepath, otu_table_filepath_db, otu_table_fname
Example #9
0
def format_biom_table(biom_table):
    """ Given a biom-format Table object, returns that Table as a BIOM string"""
    generated_by_str = "QIIME " + get_qiime_library_version()
    return biom_table.getBiomFormatJsonString(generated_by_str)
Example #10
0
def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    if opts.haiku:
        print "QIIME provides insight\nmicrobial in nature\nto ecology"
        exit(0)

    qiime_config = load_qiime_config()
    test = opts.test
    qiime_full_install = opts.qiime_full_install

    rdp_jarpath = get_rdp_jarpath()
    if rdp_jarpath is None:
        rdp_version = "Not installed."
    else:
        rdp_version = split(rdp_jarpath)[1]

    java_version = get_java_version()
    if java_version is None:
        java_version = "Not installed."

    system_info = [("Platform", platform),
                   ("Python version", python_version.replace('\n', ' ')),
                   ("Python executable", executable)]
    max_len = max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print "=================="
    for v in system_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME default reference information"
    print "==================================="
    print "For details on what files are used as QIIME's default references, see here:"
    print " https://github.com/biocore/qiime-default-reference/releases/tag/%s" % qdr_lib_version

    version_info = [("QIIME library version", get_qiime_library_version()),
                    ("QIIME script version", __version__),
                    ("qiime-default-reference version", qdr_lib_version),
                    ("NumPy version", numpy_lib_version),
                    ("SciPy version", scipy_lib_version),
                    ("pandas version", pandas_lib_version),
                    ("matplotlib version", matplotlib_lib_version),
                    ("biom-format version", biom_lib_version),
                    ("h5py version", h5py_lib_version),
                    ("qcli version", qcli_lib_version),
                    ("pyqi version", pyqi_lib_version),
                    ("scikit-bio version", skbio_lib_version),
                    ("PyNAST version", pynast_lib_version),
                    ("Emperor version", emperor_lib_version),
                    ("burrito version", burrito_lib_version),
                    ("burrito-fillings version", bfillings_lib_version),
                    ("sortmerna version", sortmerna_lib_version),
                    ("sumaclust version", sumaclust_lib_version),
                    ("swarm version", swarm_lib_version),
                    ("gdata", gdata_installed)]

    if qiime_full_install:
        version_info += [("RDP Classifier version (if installed)",
                          rdp_version),
                         ("Java version (if installed)", java_version)]

    max_len = max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print "==================="
    for v in version_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME config values"
    print "==================="
    print "For definitions of these settings and to learn how to configure QIIME, see here:"
    print " http://qiime.org/install/qiime_config.html"
    print " http://qiime.org/tutorials/parallel_qiime.html\n"
    max_len = max([len(key) for key in qiime_config])
    for key, value in qiime_config.items():
        print "%*s:\t%s" % (max_len, key, value)

    if test:
        if qiime_full_install:
            print "\nQIIME full install test results"
            print "==============================="
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyFull)
        else:
            print "\nQIIME base install test results"
            print "==============================="
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyBase)
        if opts.verbose:
            verbosity = 2
        else:
            verbosity = 1
        TextTestRunner(stream=stdout, verbosity=verbosity).run(suite)
Example #11
0
 def test_format_biom_table(self):
     """ Formatting of BIOM table correctly includes "generated-by" information
     """
     generated_by = "QIIME " + get_qiime_library_version()
     self.assertTrue(generated_by in format_biom_table(self.biom1))
    system_info = [
     ("Platform", platform),
     ("Python version",python_version.replace('\n', ' ')),
     ("Python executable",executable)]
    max_len =  max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print  "==================" 
    for v in system_info:
        print "%*s:\t%s" % (max_len,v[0],v[1])

    version_info = [
     ("PyCogent version", pycogent_lib_version),
     ("NumPy version", numpy_lib_version),
     ("matplotlib version", matplotlib_lib_version),
     ("QIIME library version", get_qiime_library_version()),
     ("QIIME script version", __version__),
     ("PyNAST version (if installed)", pynast_lib_version),
     ("RDP Classifier version (if installed)", rdp_version)]
    max_len =  max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print  "===================" 
    for v in version_info:
        print "%*s:\t%s" % (max_len,v[0],v[1])
    
    print "\nQIIME config values"
    print  "==================="    
    max_len =  max([len(key) for key in qiime_config])
    for key,value in  qiime_config.items():
        print "%*s:\t%s"%(max_len,key,value)
Example #13
0
def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    if opts.haiku:
        print "QIIME provides insight\nmicrobial in nature\nto ecology"
        exit(0)

    qiime_config = load_qiime_config()
    test = opts.test
    qiime_base_install = opts.qiime_base_install

    rdp_jarpath = get_rdp_jarpath()
    if rdp_jarpath is None:
        rdp_version = "Not installed."
    else:
        rdp_version = split(rdp_jarpath)[1]

    java_version = get_java_version()
    if java_version is None:
        java_version = "Not installed."

    system_info = [("Platform", platform),
                   ("Python version", python_version.replace('\n', ' ')),
                   ("Python executable", executable)]
    max_len = max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print "=================="
    for v in system_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    version_info = [("NumPy version", numpy_lib_version),
                    ("SciPy version", scipy_lib_version),
                    ("matplotlib version", matplotlib_lib_version),
                    ("biom-format version", biom_lib_version),
                    ("qcli version", qcli_lib_version),
                    ("pyqi version", pyqi_lib_version),
                    ("scikit-bio version", skbio_lib_version),
                    ("QIIME library version", get_qiime_library_version()),
                    ("QIIME script version", __version__),
                    ("PyNAST version (if installed)", pynast_lib_version),
                    ("Emperor version", emperor_lib_version)]
    if not qiime_base_install:
        version_info += [("RDP Classifier version (if installed)",
                          rdp_version),
                         ("Java version (if installed)", java_version)]

    max_len = max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print "==================="
    for v in version_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME config values"
    print "==================="
    max_len = max([len(key) for key in qiime_config])
    for key, value in qiime_config.items():
        print "%*s:\t%s" % (max_len, key, value)

    if test:
        if qiime_base_install:
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyBase)
        else:
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyFull)
        if opts.verbose:
            verbosity = 2
        else:
            verbosity = 1
        TextTestRunner(stream=stdout, verbosity=verbosity).run(suite)
Example #14
0
def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    if opts.haiku:
        print "QIIME provides insight\nmicrobial in nature\nto ecology"
        exit(0)

    qiime_config = load_qiime_config()
    test = opts.test
    qiime_full_install = opts.qiime_full_install

    rdp_jarpath = get_rdp_jarpath()
    if rdp_jarpath is None:
        rdp_version = "Not installed."
    else:
        rdp_version = split(rdp_jarpath)[1]

    java_version = get_java_version()
    if java_version is None:
        java_version = "Not installed."

    system_info = [
        ("Platform", platform),
        ("Python version", python_version.replace('\n', ' ')),
        ("Python executable", executable)]
    max_len = max([len(e[0]) for e in system_info])
    print "\nSystem information"
    print "=================="
    for v in system_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME default reference information"
    print "==================================="
    print "For details on what files are used as QIIME's default references, see here:"
    print " https://github.com/biocore/qiime-default-reference/releases/tag/%s" % qdr_lib_version

    version_info = [
        ("QIIME library version", get_qiime_library_version()),
        ("QIIME script version", __version__),
        ("qiime-default-reference version", qdr_lib_version),
        ("NumPy version", numpy_lib_version),
        ("SciPy version", scipy_lib_version),
        ("pandas version", pandas_lib_version),
        ("matplotlib version", matplotlib_lib_version),
        ("biom-format version", biom_lib_version),
        ("h5py version", h5py_lib_version),
        ("qcli version", qcli_lib_version),
        ("pyqi version", pyqi_lib_version),
        ("scikit-bio version", skbio_lib_version),
        ("PyNAST version", pynast_lib_version),
        ("Emperor version", emperor_lib_version),
        ("burrito version", burrito_lib_version),
        ("burrito-fillings version", bfillings_lib_version),
        ("sortmerna version", sortmerna_lib_version),
        ("sumaclust version", sumaclust_lib_version),
        ("swarm version", swarm_lib_version),
        ("gdata", gdata_installed)
    ]

    if qiime_full_install:
        version_info += [
            ("RDP Classifier version (if installed)", rdp_version),
            ("Java version (if installed)", java_version)]

    max_len = max([len(e[0]) for e in version_info])
    print "\nDependency versions"
    print "==================="
    for v in version_info:
        print "%*s:\t%s" % (max_len, v[0], v[1])

    print "\nQIIME config values"
    print "==================="
    print "For definitions of these settings and to learn how to configure QIIME, see here:"
    print " http://qiime.org/install/qiime_config.html"
    print " http://qiime.org/tutorials/parallel_qiime.html\n"
    max_len = max([len(key) for key in qiime_config])
    for key, value in qiime_config.items():
        print "%*s:\t%s" % (max_len, key, value)

    if test:
        if qiime_full_install:
            print "\nQIIME full install test results"
            print "==============================="
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyFull)
        else:
            print "\nQIIME base install test results"
            print "==============================="
            suite = TestLoader().loadTestsFromTestCase(QIIMEDependencyBase)
        if opts.verbose:
            verbosity = 2
        else:
            verbosity = 1
        TextTestRunner(stream=stdout, verbosity=verbosity).run(suite)