def _start_logging(self,
params,
args,
argv,
logger):
if logger == None:
self.logger = WorkflowLogger(generate_log_fp(params['master_script_log_dir']),
params={},
qiime_config=qiime_config)
close_logger_on_success = True
else:
self.logger = logger
close_logger_on_success = False
self.logger.write('Command:\n')
self.logger.write(' '.join(argv))
self.logger.write('\n\n')
log_input_md5s(self.logger,
[params[p] for p in self._input_file_parameter_ids])
return close_logger_on_success
def pick_subsampled_open_referenence_otus(
input_fp,
refseqs_fp,
output_dir,
percent_subsample,
new_ref_set_id,
command_handler,
params,
qiime_config,
prefilter_refseqs_fp=None,
run_tax_align_tree=True,
prefilter_percent_id=0.60,
min_otu_size=2,
step1_otu_map_fp=None,
step1_failures_fasta_fp=None,
parallel=False,
suppress_step4=False,
logger=None,
status_update_callback=print_to_stdout):
""" Run the data preparation steps of Qiime
The steps performed by this function are:
- Pick reference OTUs against refseqs_fp
- Subsample the failures to n sequences.
- Pick OTUs de novo on the n failures.
- Pick representative sequences for the resulting OTUs.
- Pick reference OTUs on all failures using the
representative set from step 4 as the reference set.
"""
# for now only allowing uclust for otu picking
denovo_otu_picking_method = 'uclust'
reference_otu_picking_method = 'uclust_ref'
# Prepare some variables for the later steps
input_dir, input_filename = split(input_fp)
input_basename, input_ext = splitext(input_filename)
create_dir(output_dir)
commands = []
python_exe_fp = qiime_config['python_exe_fp']
script_dir = get_qiime_scripts_dir()
if logger == None:
logger = WorkflowLogger(generate_log_fp(output_dir),
params=params,
qiime_config=qiime_config)
close_logger_on_success = True
else:
close_logger_on_success = False
log_input_md5s(
logger,
[input_fp, refseqs_fp, step1_otu_map_fp, step1_failures_fasta_fp])
# if the user has not passed a different reference collection for the pre-filter,
# used the main refseqs_fp. this is useful if the user wants to provide a smaller
# reference collection, or to use the input reference collection when running in
# iterative mode (rather than an iteration's new refseqs)
if prefilter_refseqs_fp == None:
prefilter_refseqs_fp = refseqs_fp
## Step 1: Closed-reference OTU picking on the input file (if not already complete)
if step1_otu_map_fp and step1_failures_fasta_fp:
step1_dir = '%s/step1_otus' % output_dir
create_dir(step1_dir)
logger.write("Using pre-existing reference otu map and failures.\n\n")
else:
if prefilter_percent_id != None:
prefilter_dir = '%s/prefilter_otus/' % output_dir
prefilter_otu_map_fp = \
'%s/%s_otus.txt' % (prefilter_dir,input_basename)
prefilter_failures_list_fp = '%s/%s_failures.txt' % \
(prefilter_dir,input_basename)
prefilter_pick_otu_cmd = pick_reference_otus(\
input_fp,prefilter_dir,reference_otu_picking_method,
prefilter_refseqs_fp,parallel,params,logger,prefilter_percent_id)
commands.append([('Pick Reference OTUs (prefilter)',
prefilter_pick_otu_cmd)])
prefiltered_input_fp = '%s/prefiltered_%s%s' %\
(prefilter_dir,input_basename,input_ext)
filter_fasta_cmd = 'filter_fasta.py -f %s -o %s -s %s -n' %\
(input_fp,prefiltered_input_fp,prefilter_failures_list_fp)
commands.append([('Filter prefilter failures from input',
filter_fasta_cmd)])
input_fp = prefiltered_input_fp
input_dir, input_filename = split(input_fp)
input_basename, input_ext = splitext(input_filename)
## Build the OTU picking command
step1_dir = \
'%s/step1_otus' % output_dir
step1_otu_map_fp = \
'%s/%s_otus.txt' % (step1_dir,input_basename)
step1_pick_otu_cmd = pick_reference_otus(\
input_fp,step1_dir,reference_otu_picking_method,
refseqs_fp,parallel,params,logger)
commands.append([('Pick Reference OTUs', step1_pick_otu_cmd)])
## Build the failures fasta file
step1_failures_list_fp = '%s/%s_failures.txt' % \
(step1_dir,input_basename)
step1_failures_fasta_fp = \
'%s/failures.fasta' % step1_dir
step1_filter_fasta_cmd = 'filter_fasta.py -f %s -s %s -o %s' %\
(input_fp,step1_failures_list_fp,step1_failures_fasta_fp)
commands.append([('Generate full failures fasta file',
step1_filter_fasta_cmd)])
# Call the command handler on the list of commands
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
step1_repset_fasta_fp = \
'%s/step1_rep_set.fna' % step1_dir
step1_pick_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step1_otu_map_fp, step1_repset_fasta_fp, input_fp)
commands.append([('Pick rep set', step1_pick_rep_set_cmd)])
## Subsample the failures fasta file to retain (roughly) the
## percent_subsample
step2_input_fasta_fp = \
'%s/subsampled_failures.fasta' % step1_dir
subsample_fasta(step1_failures_fasta_fp, step2_input_fasta_fp,
percent_subsample)
## Prep the OTU picking command for the subsampled failures
step2_dir = '%s/step2_otus/' % output_dir
step2_cmd = pick_denovo_otus(step2_input_fasta_fp, step2_dir,
new_ref_set_id, denovo_otu_picking_method,
params, logger)
step2_otu_map_fp = '%s/subsampled_failures_otus.txt' % step2_dir
commands.append([('Pick de novo OTUs for new clusters', step2_cmd)])
## Prep the rep set picking command for the subsampled failures
step2_repset_fasta_fp = '%s/step2_rep_set.fna' % step2_dir
step2_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step2_otu_map_fp,step2_repset_fasta_fp,step2_input_fasta_fp)
commands.append([('Pick representative set for subsampled failures',
step2_rep_set_cmd)])
step3_dir = '%s/step3_otus/' % output_dir
step3_otu_map_fp = '%s/failures_otus.txt' % step3_dir
step3_failures_list_fp = '%s/failures_failures.txt' % step3_dir
step3_cmd = pick_reference_otus(step1_failures_fasta_fp, step3_dir,
reference_otu_picking_method,
step2_repset_fasta_fp, parallel, params,
logger)
commands.append([('Pick reference OTUs using de novo rep set', step3_cmd)])
# name the final otu map
merged_otu_map_fp = '%s/final_otu_map.txt' % output_dir
if not suppress_step4:
step3_failures_fasta_fp = '%s/failures_failures.fasta' % step3_dir
step3_filter_fasta_cmd = 'filter_fasta.py -f %s -s %s -o %s' %\
(step1_failures_fasta_fp,step3_failures_list_fp,step3_failures_fasta_fp)
commands.append([('Create fasta file of step3 failures',
step3_filter_fasta_cmd)])
step4_dir = '%s/step4_otus/' % output_dir
step4_cmd = pick_denovo_otus(step3_failures_fasta_fp, step4_dir,
'.'.join([new_ref_set_id, 'CleanUp']),
denovo_otu_picking_method, params, logger)
step4_otu_map_fp = '%s/failures_failures_otus.txt' % step4_dir
commands.append([('Pick de novo OTUs on step3 failures', step4_cmd)])
# Merge the otu maps
cat_otu_tables_cmd = 'cat %s %s %s >> %s' %\
(step1_otu_map_fp,step3_otu_map_fp,step4_otu_map_fp,merged_otu_map_fp)
commands.append([('Merge OTU maps', cat_otu_tables_cmd)])
step4_repset_fasta_fp = '%s/step4_rep_set.fna' % step4_dir
step4_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step4_otu_map_fp,step4_repset_fasta_fp,step3_failures_fasta_fp)
commands.append([('Pick representative set for subsampled failures',
step4_rep_set_cmd)])
else:
# Merge the otu maps
cat_otu_tables_cmd = 'cat %s %s >> %s' %\
(step1_otu_map_fp,step3_otu_map_fp,merged_otu_map_fp)
commands.append([('Merge OTU maps', cat_otu_tables_cmd)])
# Move the step 3 failures file to the top-level directory
commands.append([('Move final failures file to top-level directory',
'mv %s %s/final_failures.txt' %
(step3_failures_list_fp, output_dir))])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
otu_fp = merged_otu_map_fp
# Filter singletons from the otu map
otu_no_singletons_fp = '%s/final_otu_map_mc%d.txt' % (output_dir,
min_otu_size)
otus_to_keep = filter_otus_from_otu_map(otu_fp, otu_no_singletons_fp,
min_otu_size)
## make the final representative seqs file and a new refseqs file that
## could be used in subsequent otu picking runs.
## this is clunky. first, we need to do this without singletons to match
## the otu map without singletons. next, there is a difference in what
## we need the reference set to be and what we need the repseqs to be.
## the reference set needs to be a superset of the input reference set
## to this set. the repset needs to be only the sequences that were observed
## in this data set, and we want reps for the step1 reference otus to be
## reads from this run so we don't hit issues building a tree using
## sequences of very different lengths. so...
final_repset_fp = '%s/rep_set.fna' % output_dir
final_repset_f = open(final_repset_fp, 'w')
new_refseqs_fp = '%s/new_refseqs.fna' % output_dir
# write non-singleton otus representative sequences from step1 to the
# final rep set file
for otu_id, seq in MinimalFastaParser(open(step1_repset_fasta_fp, 'U')):
if otu_id.split()[0] in otus_to_keep:
final_repset_f.write('>%s\n%s\n' % (otu_id, seq))
# copy the full input refseqs file to the new refseqs_fp
copy(refseqs_fp, new_refseqs_fp)
new_refseqs_f = open(new_refseqs_fp, 'a')
new_refseqs_f.write('\n')
# iterate over all representative sequences from step2 and step4 and write
# those corresponding to non-singleton otus to the final representative set
# file and the new reference sequences file.
for otu_id, seq in MinimalFastaParser(open(step2_repset_fasta_fp, 'U')):
if otu_id.split()[0] in otus_to_keep:
new_refseqs_f.write('>%s\n%s\n' % (otu_id, seq))
final_repset_f.write('>%s\n%s\n' % (otu_id, seq))
if not suppress_step4:
for otu_id, seq in MinimalFastaParser(open(step4_repset_fasta_fp,
'U')):
if otu_id.split()[0] in otus_to_keep:
new_refseqs_f.write('>%s\n%s\n' % (otu_id, seq))
final_repset_f.write('>%s\n%s\n' % (otu_id, seq))
new_refseqs_f.close()
final_repset_f.close()
# Prep the make_otu_table.py command
otu_table_fp = '%s/otu_table_mc%d.biom' % (output_dir, min_otu_size)
make_otu_table_cmd = 'make_otu_table.py -i %s -o %s' %\
(otu_no_singletons_fp,otu_table_fp)
commands.append([("Make the otu table", make_otu_table_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
if run_tax_align_tree:
taxonomy_fp, pynast_failures_fp = tax_align_tree(
repset_fasta_fp=final_repset_fp,
output_dir=output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
parallel=parallel,
logger=logger,
status_update_callback=status_update_callback)
# Add taxa to otu table
otu_table_w_tax_fp = \
'%s/otu_table_mc%d_w_tax.biom' % (output_dir,min_otu_size)
add_taxa_cmd = 'add_taxa.py -i %s -t %s -o %s' %\
(otu_table_fp,taxonomy_fp,otu_table_w_tax_fp)
commands.append([("Add taxa to OTU table", add_taxa_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
# Build OTU table without PyNAST failures
otu_table_fp = \
'%s/otu_table_mc%d_w_tax_no_pynast_failures.biom' % (output_dir,min_otu_size)
filtered_otu_table = filter_otus_from_otu_table(
parse_biom_table(open(otu_table_w_tax_fp, 'U')),
get_seq_ids_from_fasta_file(open(pynast_failures_fp, 'U')),
0,
inf,
0,
inf,
negate_ids_to_keep=True)
otu_table_f = open(otu_table_fp, 'w')
otu_table_f.write(format_biom_table(filtered_otu_table))
otu_table_f.close()
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=close_logger_on_success)
def iterative_pick_subsampled_open_referenence_otus(
input_fps,
refseqs_fp,
output_dir,
percent_subsample,
new_ref_set_id,
command_handler,
params,
qiime_config,
prefilter_refseqs_fp=None,
prefilter_percent_id=0.60,
min_otu_size=2,
run_tax_align_tree=True,
step1_otu_map_fp=None,
step1_failures_fasta_fp=None,
parallel=False,
suppress_step4=False,
logger=None,
status_update_callback=print_to_stdout):
""" Call the pick_subsampled_open_referenence_otus workflow on multiple inputs
and handle processing of the results.
"""
create_dir(output_dir)
commands = []
if logger == None:
logger = WorkflowLogger(generate_log_fp(output_dir),
params=params,
qiime_config=qiime_config)
close_logger_on_success = True
else:
close_logger_on_success = False
# if the user has not passed a different reference collection for the pre-filter,
# used the input refseqs_fp for all iterations. we want to pre-filter all data against
# the input data as lower percent identity searches with uclust can be slow, so we
# want the reference collection to stay at a reasonable size.
if prefilter_refseqs_fp == None:
prefilter_refseqs_fp = refseqs_fp
otu_table_fps = []
repset_fasta_fps = []
for i, input_fp in enumerate(input_fps):
iteration_output_dir = '%s/%d/' % (output_dir, i)
if iteration_output_exists(iteration_output_dir, min_otu_size):
# if the output from an iteration already exists, skip that
# iteration (useful for continuing failed runs)
log_input_md5s(logger, [input_fp, refseqs_fp])
logger.write(
'Iteration %d (input file: %s) output data already exists. '
'Skipping and moving to next.\n\n' % (i, input_fp))
else:
pick_subsampled_open_referenence_otus(
input_fp=input_fp,
refseqs_fp=refseqs_fp,
output_dir=iteration_output_dir,
percent_subsample=percent_subsample,
new_ref_set_id='.'.join([new_ref_set_id,
str(i)]),
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
run_tax_align_tree=False,
prefilter_refseqs_fp=prefilter_refseqs_fp,
prefilter_percent_id=prefilter_percent_id,
min_otu_size=min_otu_size,
step1_otu_map_fp=step1_otu_map_fp,
step1_failures_fasta_fp=step1_failures_fasta_fp,
parallel=parallel,
suppress_step4=suppress_step4,
logger=logger,
status_update_callback=status_update_callback)
## perform post-iteration file shuffling whether the previous iteration's
## data previously existed or was just computed.
# step1 otu map and failures can only be used for the first iteration
# as subsequent iterations need to use updated refseqs files
step1_otu_map_fp = step1_failures_fasta_fp = None
new_refseqs_fp = '%s/new_refseqs.fna' % iteration_output_dir
refseqs_fp = new_refseqs_fp
otu_table_fps.append('%s/otu_table_mc%d.biom' %
(iteration_output_dir, min_otu_size))
repset_fasta_fps.append('%s/rep_set.fna' % iteration_output_dir)
# Merge OTU tables - check for existence first as this step has historically
# been a frequent failure, so is sometimes run manually in failed runs.
otu_table_fp = '%s/otu_table_mc%d.biom' % (output_dir, min_otu_size)
if not (exists(otu_table_fp) and getsize(otu_table_fp) > 0):
merge_cmd = 'merge_otu_tables.py -i %s -o %s' %\
(','.join(otu_table_fps),otu_table_fp)
commands.append([("Merge OTU tables", merge_cmd)])
# Build master rep set
final_repset_fp = '%s/rep_set.fna' % output_dir
final_repset_from_iteration_repsets_fps(repset_fasta_fps, final_repset_fp)
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
if run_tax_align_tree:
otu_table_w_tax_fp = \
'%s/otu_table_mc%d_w_tax.biom' % (output_dir,min_otu_size)
final_otu_table_fp = \
'%s/otu_table_mc%d_w_tax_no_pynast_failures.biom' % (output_dir,min_otu_size)
if exists(final_otu_table_fp) and getsize(final_otu_table_fp) > 0:
logger.write("Final output file exists (%s). Will not rebuild." %
otu_table_fp)
else:
# remove files from partially completed runs
remove_files([otu_table_w_tax_fp, final_otu_table_fp],
error_on_missing=False)
taxonomy_fp, pynast_failures_fp = tax_align_tree(
repset_fasta_fp=final_repset_fp,
output_dir=output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
parallel=parallel,
logger=logger,
status_update_callback=status_update_callback)
# Add taxa to otu table
add_taxa_cmd = 'add_taxa.py -i %s -t %s -o %s' %\
(otu_table_fp,taxonomy_fp,otu_table_w_tax_fp)
commands.append([("Add taxa to OTU table", add_taxa_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
# Build OTU table without PyNAST failures
filtered_otu_table = filter_otus_from_otu_table(
parse_biom_table(open(otu_table_w_tax_fp, 'U')),
get_seq_ids_from_fasta_file(open(pynast_failures_fp, 'U')),
0,
inf,
0,
inf,
negate_ids_to_keep=True)
otu_table_f = open(final_otu_table_fp, 'w')
otu_table_f.write(format_biom_table(filtered_otu_table))
otu_table_f.close()
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
logger.close()
def run_core_diversity_analyses(
biom_fp,
mapping_fp,
sampling_depth,
output_dir,
qiime_config,
command_handler=call_commands_serially,
tree_fp=None,
params=None,
categories=None,
arare_min_rare_depth=10,
arare_num_steps=10,
parallel=False,
status_update_callback=print_to_stdout):
"""
"""
if categories != None:
# Validate categories provided by the users
mapping_data, mapping_comments = \
parse_mapping_file_to_dict(open(mapping_fp,'U'))
metadata_map = MetadataMap(mapping_data, mapping_comments)
for c in categories:
if c not in metadata_map.CategoryNames:
raise ValueError, ("Category '%s' is not a column header "
"in your mapping file. "
"Categories are case and white space sensitive. Valid "
"choices are: (%s)" % (c,', '.join(metadata_map.CategoryNames)))
if metadata_map.hasSingleCategoryValue(c):
raise ValueError, ("Category '%s' contains only one value. "
"Categories analyzed here require at least two values." % c)
else:
categories= []
# prep some variables
if params == None:
params = parse_qiime_parameters([])
create_dir(output_dir)
index_fp = '%s/index.html' % output_dir
index_links = []
commands = []
python_exe_fp = qiime_config['python_exe_fp']
script_dir = get_qiime_scripts_dir()
# begin logging
log_fp = generate_log_fp(output_dir)
index_links.append(('Master run log',log_fp,'Log files'))
logger = WorkflowLogger(log_fp,
params=params,
qiime_config=qiime_config)
input_fps = [biom_fp,mapping_fp]
if tree_fp != None:
input_fps.append(tree_fp)
log_input_md5s(logger,input_fps)
bdiv_even_output_dir = '%s/bdiv_even%d/' % (output_dir,sampling_depth)
even_dm_fps = run_beta_diversity_through_plots(
otu_table_fp=biom_fp,
mapping_fp=mapping_fp,
output_dir=bdiv_even_output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
sampling_depth=sampling_depth,
# force suppression of distance histograms - boxplots work better
# in this context, and are created below.
histogram_categories=[],
tree_fp=tree_fp,
parallel=parallel,
logger=logger,
status_update_callback=status_update_callback)
for bdiv_metric, dm_fp in even_dm_fps:
for category in categories:
boxplots_output_dir = '%s/%s_boxplots/' % (bdiv_even_output_dir,bdiv_metric)
try:
params_str = get_params_str(params['make_distance_boxplots'])
except KeyError:
params_str = ''
boxplots_cmd = \
'make_distance_boxplots.py -d %s -f %s -o %s -m %s -n 999 %s' %\
(dm_fp, category, boxplots_output_dir, mapping_fp, params_str)
commands.append([('Boxplots (%s)' % category,
boxplots_cmd)])
index_links.append(('Distance boxplots (%s)' % bdiv_metric,
'%s/%s_Distances.pdf' % \
(boxplots_output_dir,category),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('Distance boxplots statistics (%s)' % bdiv_metric,
'%s/%s_Stats.txt' % \
(boxplots_output_dir,category),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('3D plot (%s, continuous coloring)' % bdiv_metric,
'%s/%s_3d_continuous/%s_pc_3D_PCoA_plots.html' % \
(bdiv_even_output_dir,bdiv_metric,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('3D plot (%s, discrete coloring)' % bdiv_metric,
'%s/%s_3d_discrete/%s_pc_3D_PCoA_plots.html' % \
(bdiv_even_output_dir,bdiv_metric,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('2D plot (%s, continuous coloring)' % bdiv_metric,
'%s/%s_2d_continuous/%s_pc_2D_PCoA_plots.html' % \
(bdiv_even_output_dir,bdiv_metric,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('2D plot (%s, discrete coloring)' % bdiv_metric,
'%s/%s_2d_discrete/%s_pc_2D_PCoA_plots.html' % \
(bdiv_even_output_dir,bdiv_metric,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('Distance matrix (%s)' % bdiv_metric,
'%s/%s_dm.txt' % \
(bdiv_even_output_dir,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
index_links.append(('Principal coordinate matrix (%s)' % bdiv_metric,
'%s/%s_pc.txt' % \
(bdiv_even_output_dir,bdiv_metric),
'Beta diversity results (even sampling: %d)' % sampling_depth))
## Alpha rarefaction workflow
arare_full_output_dir = '%s/arare_max%d/' % (output_dir,sampling_depth)
run_qiime_alpha_rarefaction(
otu_table_fp=biom_fp,
mapping_fp=mapping_fp,
output_dir=arare_full_output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
tree_fp=tree_fp,
num_steps=arare_num_steps,
parallel=parallel,
logger=logger,
min_rare_depth=arare_min_rare_depth,
max_rare_depth=sampling_depth,
status_update_callback=status_update_callback)
index_links.append(('Alpha rarefaction plots',
'%s/alpha_rarefaction_plots/rarefaction_plots.html'\
% arare_full_output_dir,
"Alpha rarefaction results"))
collated_alpha_diversity_fps = \
glob('%s/alpha_div_collated/*txt' % arare_full_output_dir)
try:
params_str = get_params_str(params['compare_alpha_diversity'])
except KeyError:
params_str = ''
for c in categories:
for collated_alpha_diversity_fp in collated_alpha_diversity_fps:
alpha_metric = splitext(split(collated_alpha_diversity_fp)[1])[0]
alpha_comparison_output_fp = '%s/%s_%s.txt' % \
(arare_full_output_dir,c,alpha_metric)
compare_alpha_cmd = \
'compare_alpha_diversity.py -i %s -m %s -c %s -d %s -o %s -n 999 %s' %\
(collated_alpha_diversity_fp, mapping_fp, c,
sampling_depth, alpha_comparison_output_fp, params_str)
commands.append([('Compare alpha diversity (%s, %s)' %\
(category,alpha_metric),
compare_alpha_cmd)])
index_links.append(
('Alpha diversity statistics (%s, %s)' % (category,alpha_metric),
alpha_comparison_output_fp,
"Alpha rarefaction results"))
taxa_plots_output_dir = '%s/taxa_plots/' % output_dir
run_summarize_taxa_through_plots(
otu_table_fp=biom_fp,
mapping_fp=mapping_fp,
output_dir=taxa_plots_output_dir,
mapping_cat=None,
sort=True,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
logger=logger,
status_update_callback=status_update_callback)
index_links.append(('Taxa summary bar plots',
'%s/taxa_summary_plots/bar_charts.html'\
% taxa_plots_output_dir,
"Taxonomic summary results"))
index_links.append(('Taxa summary area plots',
'%s/taxa_summary_plots/area_charts.html'\
% taxa_plots_output_dir,
"Taxonomic summary results"))
for c in categories:
taxa_plots_output_dir = '%s/taxa_plots_%s/' % (output_dir,c)
run_summarize_taxa_through_plots(
otu_table_fp=biom_fp,
mapping_fp=mapping_fp,
output_dir=taxa_plots_output_dir,
mapping_cat=c,
sort=True,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
logger=logger,
status_update_callback=status_update_callback)
index_links.append(('Taxa summary bar plots',
'%s/taxa_summary_plots/bar_charts.html'\
% taxa_plots_output_dir,
"Taxonomic summary results (by %s)" % c))
index_links.append(('Taxa summary area plots',
'%s/taxa_summary_plots/area_charts.html'\
% taxa_plots_output_dir,
"Taxonomic summary results (by %s)" % c))
# OTU category significance
for category in categories:
category_signifance_fp = \
'%s/category_significance_%s.txt' % (output_dir, category)
try:
params_str = get_params_str(params['otu_category_significance'])
except KeyError:
params_str = ''
# Build the OTU cateogry significance command
category_significance_cmd = \
'otu_category_significance.py -i %s -m %s -c %s -o %s %s' %\
(biom_fp, mapping_fp, category,
category_signifance_fp, params_str)
commands.append([('OTU category significance (%s)' % category,
category_significance_cmd)])
index_links.append(('Category significance (%s)' % category,
category_signifance_fp,
"Category results"))
command_handler(commands, status_update_callback, logger)
generate_index_page(index_links,index_fp)
def pick_subsampled_open_referenence_otus(input_fp,
refseqs_fp,
output_dir,
percent_subsample,
new_ref_set_id,
command_handler,
params,
qiime_config,
prefilter_refseqs_fp=None,
run_tax_align_tree=True,
prefilter_percent_id=0.60,
min_otu_size=2,
step1_otu_map_fp=None,
step1_failures_fasta_fp=None,
parallel=False,
suppress_step4=False,
logger=None,
status_update_callback=print_to_stdout):
""" Run the data preparation steps of Qiime
The steps performed by this function are:
- Pick reference OTUs against refseqs_fp
- Subsample the failures to n sequences.
- Pick OTUs de novo on the n failures.
- Pick representative sequences for the resulting OTUs.
- Pick reference OTUs on all failures using the
representative set from step 4 as the reference set.
"""
# for now only allowing uclust for otu picking
denovo_otu_picking_method = 'uclust'
reference_otu_picking_method = 'uclust_ref'
# Prepare some variables for the later steps
input_dir, input_filename = split(input_fp)
input_basename, input_ext = splitext(input_filename)
create_dir(output_dir)
commands = []
python_exe_fp = qiime_config['python_exe_fp']
script_dir = get_qiime_scripts_dir()
if logger == None:
logger = WorkflowLogger(generate_log_fp(output_dir),
params=params,
qiime_config=qiime_config)
close_logger_on_success = True
else:
close_logger_on_success = False
log_input_md5s(logger,[input_fp,refseqs_fp,step1_otu_map_fp,step1_failures_fasta_fp])
# if the user has not passed a different reference collection for the pre-filter,
# used the main refseqs_fp. this is useful if the user wants to provide a smaller
# reference collection, or to use the input reference collection when running in
# iterative mode (rather than an iteration's new refseqs)
if prefilter_refseqs_fp == None:
prefilter_refseqs_fp = refseqs_fp
## Step 1: Closed-reference OTU picking on the input file (if not already complete)
if step1_otu_map_fp and step1_failures_fasta_fp:
step1_dir = '%s/step1_otus' % output_dir
create_dir(step1_dir)
logger.write("Using pre-existing reference otu map and failures.\n\n")
else:
if prefilter_percent_id != None:
prefilter_dir = '%s/prefilter_otus/' % output_dir
prefilter_otu_map_fp = \
'%s/%s_otus.txt' % (prefilter_dir,input_basename)
prefilter_failures_list_fp = '%s/%s_failures.txt' % \
(prefilter_dir,input_basename)
prefilter_pick_otu_cmd = pick_reference_otus(\
input_fp,prefilter_dir,reference_otu_picking_method,
prefilter_refseqs_fp,parallel,params,logger,prefilter_percent_id)
commands.append([('Pick Reference OTUs (prefilter)', prefilter_pick_otu_cmd)])
prefiltered_input_fp = '%s/prefiltered_%s%s' %\
(prefilter_dir,input_basename,input_ext)
filter_fasta_cmd = 'filter_fasta.py -f %s -o %s -s %s -n' %\
(input_fp,prefiltered_input_fp,prefilter_failures_list_fp)
commands.append([('Filter prefilter failures from input', filter_fasta_cmd)])
input_fp = prefiltered_input_fp
input_dir, input_filename = split(input_fp)
input_basename, input_ext = splitext(input_filename)
## Build the OTU picking command
step1_dir = \
'%s/step1_otus' % output_dir
step1_otu_map_fp = \
'%s/%s_otus.txt' % (step1_dir,input_basename)
step1_pick_otu_cmd = pick_reference_otus(\
input_fp,step1_dir,reference_otu_picking_method,
refseqs_fp,parallel,params,logger)
commands.append([('Pick Reference OTUs', step1_pick_otu_cmd)])
## Build the failures fasta file
step1_failures_list_fp = '%s/%s_failures.txt' % \
(step1_dir,input_basename)
step1_failures_fasta_fp = \
'%s/failures.fasta' % step1_dir
step1_filter_fasta_cmd = 'filter_fasta.py -f %s -s %s -o %s' %\
(input_fp,step1_failures_list_fp,step1_failures_fasta_fp)
commands.append([('Generate full failures fasta file',
step1_filter_fasta_cmd)])
# Call the command handler on the list of commands
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
step1_repset_fasta_fp = \
'%s/step1_rep_set.fna' % step1_dir
step1_pick_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step1_otu_map_fp, step1_repset_fasta_fp, input_fp)
commands.append([('Pick rep set',step1_pick_rep_set_cmd)])
## Subsample the failures fasta file to retain (roughly) the
## percent_subsample
step2_input_fasta_fp = \
'%s/subsampled_failures.fasta' % step1_dir
subsample_fasta(step1_failures_fasta_fp,
step2_input_fasta_fp,
percent_subsample)
## Prep the OTU picking command for the subsampled failures
step2_dir = '%s/step2_otus/' % output_dir
step2_cmd = pick_denovo_otus(step2_input_fasta_fp,
step2_dir,
new_ref_set_id,
denovo_otu_picking_method,
params,
logger)
step2_otu_map_fp = '%s/subsampled_failures_otus.txt' % step2_dir
commands.append([('Pick de novo OTUs for new clusters', step2_cmd)])
## Prep the rep set picking command for the subsampled failures
step2_repset_fasta_fp = '%s/step2_rep_set.fna' % step2_dir
step2_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step2_otu_map_fp,step2_repset_fasta_fp,step2_input_fasta_fp)
commands.append([('Pick representative set for subsampled failures',step2_rep_set_cmd)])
step3_dir = '%s/step3_otus/' % output_dir
step3_otu_map_fp = '%s/failures_otus.txt' % step3_dir
step3_failures_list_fp = '%s/failures_failures.txt' % step3_dir
step3_cmd = pick_reference_otus(
step1_failures_fasta_fp,
step3_dir,
reference_otu_picking_method,
step2_repset_fasta_fp,
parallel,
params,
logger)
commands.append([
('Pick reference OTUs using de novo rep set',step3_cmd)])
# name the final otu map
merged_otu_map_fp = '%s/final_otu_map.txt' % output_dir
if not suppress_step4:
step3_failures_fasta_fp = '%s/failures_failures.fasta' % step3_dir
step3_filter_fasta_cmd = 'filter_fasta.py -f %s -s %s -o %s' %\
(step1_failures_fasta_fp,step3_failures_list_fp,step3_failures_fasta_fp)
commands.append([('Create fasta file of step3 failures',
step3_filter_fasta_cmd)])
step4_dir = '%s/step4_otus/' % output_dir
step4_cmd = pick_denovo_otus(step3_failures_fasta_fp,
step4_dir,
'.'.join([new_ref_set_id,'CleanUp']),
denovo_otu_picking_method,
params,
logger)
step4_otu_map_fp = '%s/failures_failures_otus.txt' % step4_dir
commands.append([('Pick de novo OTUs on step3 failures', step4_cmd)])
# Merge the otu maps
cat_otu_tables_cmd = 'cat %s %s %s >> %s' %\
(step1_otu_map_fp,step3_otu_map_fp,step4_otu_map_fp,merged_otu_map_fp)
commands.append([('Merge OTU maps',cat_otu_tables_cmd)])
step4_repset_fasta_fp = '%s/step4_rep_set.fna' % step4_dir
step4_rep_set_cmd = 'pick_rep_set.py -i %s -o %s -f %s' %\
(step4_otu_map_fp,step4_repset_fasta_fp,step3_failures_fasta_fp)
commands.append([('Pick representative set for subsampled failures',step4_rep_set_cmd)])
else:
# Merge the otu maps
cat_otu_tables_cmd = 'cat %s %s >> %s' %\
(step1_otu_map_fp,step3_otu_map_fp,merged_otu_map_fp)
commands.append([('Merge OTU maps',cat_otu_tables_cmd)])
# Move the step 3 failures file to the top-level directory
commands.append([('Move final failures file to top-level directory',
'mv %s %s/final_failures.txt' % (step3_failures_list_fp,output_dir))])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
otu_fp = merged_otu_map_fp
# Filter singletons from the otu map
otu_no_singletons_fp = '%s/final_otu_map_mc%d.txt' % (output_dir,min_otu_size)
otus_to_keep = filter_otus_from_otu_map(otu_fp,otu_no_singletons_fp,min_otu_size)
## make the final representative seqs file and a new refseqs file that
## could be used in subsequent otu picking runs.
## this is clunky. first, we need to do this without singletons to match
## the otu map without singletons. next, there is a difference in what
## we need the reference set to be and what we need the repseqs to be.
## the reference set needs to be a superset of the input reference set
## to this set. the repset needs to be only the sequences that were observed
## in this data set, and we want reps for the step1 reference otus to be
## reads from this run so we don't hit issues building a tree using
## sequences of very different lengths. so...
final_repset_fp = '%s/rep_set.fna' % output_dir
final_repset_f = open(final_repset_fp,'w')
new_refseqs_fp = '%s/new_refseqs.fna' % output_dir
# write non-singleton otus representative sequences from step1 to the
# final rep set file
for otu_id, seq in MinimalFastaParser(open(step1_repset_fasta_fp,'U')):
if otu_id.split()[0] in otus_to_keep:
final_repset_f.write('>%s\n%s\n' % (otu_id,seq))
# copy the full input refseqs file to the new refseqs_fp
copy(refseqs_fp,new_refseqs_fp)
new_refseqs_f = open(new_refseqs_fp,'a')
new_refseqs_f.write('\n')
# iterate over all representative sequences from step2 and step4 and write
# those corresponding to non-singleton otus to the final representative set
# file and the new reference sequences file.
for otu_id, seq in MinimalFastaParser(open(step2_repset_fasta_fp,'U')):
if otu_id.split()[0] in otus_to_keep:
new_refseqs_f.write('>%s\n%s\n' % (otu_id,seq))
final_repset_f.write('>%s\n%s\n' % (otu_id,seq))
if not suppress_step4:
for otu_id, seq in MinimalFastaParser(open(step4_repset_fasta_fp,'U')):
if otu_id.split()[0] in otus_to_keep:
new_refseqs_f.write('>%s\n%s\n' % (otu_id,seq))
final_repset_f.write('>%s\n%s\n' % (otu_id,seq))
new_refseqs_f.close()
final_repset_f.close()
# Prep the make_otu_table.py command
otu_table_fp = '%s/otu_table_mc%d.biom' % (output_dir,min_otu_size)
make_otu_table_cmd = 'make_otu_table.py -i %s -o %s' %\
(otu_no_singletons_fp,otu_table_fp)
commands.append([("Make the otu table",make_otu_table_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
if run_tax_align_tree:
taxonomy_fp, pynast_failures_fp = tax_align_tree(
repset_fasta_fp=final_repset_fp,
output_dir=output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
parallel=parallel,
logger=logger,
status_update_callback=status_update_callback)
# Add taxa to otu table
otu_table_w_tax_fp = \
'%s/otu_table_mc%d_w_tax.biom' % (output_dir,min_otu_size)
add_taxa_cmd = 'add_taxa.py -i %s -t %s -o %s' %\
(otu_table_fp,taxonomy_fp,otu_table_w_tax_fp)
commands.append([("Add taxa to OTU table",add_taxa_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
# Build OTU table without PyNAST failures
otu_table_fp = \
'%s/otu_table_mc%d_w_tax_no_pynast_failures.biom' % (output_dir,min_otu_size)
filtered_otu_table = filter_otus_from_otu_table(
parse_biom_table(open(otu_table_w_tax_fp,'U')),
get_seq_ids_from_fasta_file(open(pynast_failures_fp,'U')),
0,inf,0,inf,negate_ids_to_keep=True)
otu_table_f = open(otu_table_fp,'w')
otu_table_f.write(format_biom_table(filtered_otu_table))
otu_table_f.close()
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=close_logger_on_success)
def iterative_pick_subsampled_open_referenence_otus(
input_fps,
refseqs_fp,
output_dir,
percent_subsample,
new_ref_set_id,
command_handler,
params,
qiime_config,
prefilter_refseqs_fp=None,
prefilter_percent_id=0.60,
min_otu_size=2,
run_tax_align_tree=True,
step1_otu_map_fp=None,
step1_failures_fasta_fp=None,
parallel=False,
suppress_step4=False,
logger=None,
status_update_callback=print_to_stdout):
""" Call the pick_subsampled_open_referenence_otus workflow on multiple inputs
and handle processing of the results.
"""
create_dir(output_dir)
commands = []
if logger == None:
logger = WorkflowLogger(generate_log_fp(output_dir),
params=params,
qiime_config=qiime_config)
close_logger_on_success = True
else:
close_logger_on_success = False
# if the user has not passed a different reference collection for the pre-filter,
# used the input refseqs_fp for all iterations. we want to pre-filter all data against
# the input data as lower percent identity searches with uclust can be slow, so we
# want the reference collection to stay at a reasonable size.
if prefilter_refseqs_fp == None:
prefilter_refseqs_fp = refseqs_fp
otu_table_fps = []
repset_fasta_fps = []
for i,input_fp in enumerate(input_fps):
iteration_output_dir = '%s/%d/' % (output_dir,i)
if iteration_output_exists(iteration_output_dir,min_otu_size):
# if the output from an iteration already exists, skip that
# iteration (useful for continuing failed runs)
log_input_md5s(logger,[input_fp,refseqs_fp])
logger.write('Iteration %d (input file: %s) output data already exists. '
'Skipping and moving to next.\n\n' % (i,input_fp))
else:
pick_subsampled_open_referenence_otus(input_fp=input_fp,
refseqs_fp=refseqs_fp,
output_dir=iteration_output_dir,
percent_subsample=percent_subsample,
new_ref_set_id='.'.join([new_ref_set_id,str(i)]),
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
run_tax_align_tree=False,
prefilter_refseqs_fp=prefilter_refseqs_fp,
prefilter_percent_id=prefilter_percent_id,
min_otu_size=min_otu_size,
step1_otu_map_fp=step1_otu_map_fp,
step1_failures_fasta_fp=step1_failures_fasta_fp,
parallel=parallel,
suppress_step4=suppress_step4,
logger=logger,
status_update_callback=status_update_callback)
## perform post-iteration file shuffling whether the previous iteration's
## data previously existed or was just computed.
# step1 otu map and failures can only be used for the first iteration
# as subsequent iterations need to use updated refseqs files
step1_otu_map_fp = step1_failures_fasta_fp = None
new_refseqs_fp = '%s/new_refseqs.fna' % iteration_output_dir
refseqs_fp = new_refseqs_fp
otu_table_fps.append('%s/otu_table_mc%d.biom' % (iteration_output_dir,min_otu_size))
repset_fasta_fps.append('%s/rep_set.fna' % iteration_output_dir)
# Merge OTU tables - check for existence first as this step has historically
# been a frequent failure, so is sometimes run manually in failed runs.
otu_table_fp = '%s/otu_table_mc%d.biom' % (output_dir,min_otu_size)
if not (exists(otu_table_fp) and getsize(otu_table_fp) > 0):
merge_cmd = 'merge_otu_tables.py -i %s -o %s' %\
(','.join(otu_table_fps),otu_table_fp)
commands.append([("Merge OTU tables",merge_cmd)])
# Build master rep set
final_repset_fp = '%s/rep_set.fna' % output_dir
final_repset_from_iteration_repsets_fps(repset_fasta_fps,final_repset_fp)
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
if run_tax_align_tree:
otu_table_w_tax_fp = \
'%s/otu_table_mc%d_w_tax.biom' % (output_dir,min_otu_size)
final_otu_table_fp = \
'%s/otu_table_mc%d_w_tax_no_pynast_failures.biom' % (output_dir,min_otu_size)
if exists(final_otu_table_fp) and getsize(final_otu_table_fp) > 0:
logger.write("Final output file exists (%s). Will not rebuild." % otu_table_fp)
else:
# remove files from partially completed runs
remove_files([otu_table_w_tax_fp,final_otu_table_fp],error_on_missing=False)
taxonomy_fp, pynast_failures_fp = tax_align_tree(
repset_fasta_fp=final_repset_fp,
output_dir=output_dir,
command_handler=command_handler,
params=params,
qiime_config=qiime_config,
parallel=parallel,
logger=logger,
status_update_callback=status_update_callback)
# Add taxa to otu table
add_taxa_cmd = 'add_taxa.py -i %s -t %s -o %s' %\
(otu_table_fp,taxonomy_fp,otu_table_w_tax_fp)
commands.append([("Add taxa to OTU table",add_taxa_cmd)])
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
# Build OTU table without PyNAST failures
filtered_otu_table = filter_otus_from_otu_table(
parse_biom_table(open(otu_table_w_tax_fp,'U')),
get_seq_ids_from_fasta_file(open(pynast_failures_fp,'U')),
0,inf,0,inf,negate_ids_to_keep=True)
otu_table_f = open(final_otu_table_fp,'w')
otu_table_f.write(format_biom_table(filtered_otu_table))
otu_table_f.close()
command_handler(commands,
status_update_callback,
logger=logger,
close_logger_on_success=False)
commands = []
logger.close()