def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
#set some defaults for the options
input_dir = opts.input_dir
output_dir = opts.output_dir or input_dir
tmp_dir = opts.tmp_dir or output_dir
parallel_method = opts.parallel_method
asr_method = opts.asr_method
predict_traits_method = opts.prediction_method
if opts.num_jobs > 20 and parallel_method == 'multithreaded':
raise ValueError(
'You probably dont want to run multithreaded evaluations with a large num_jobs. Please adjust options num_jobs and or parallel_method'
)
if opts.with_confidence and asr_method not in ['ace_ml', 'ace_reml']:
raise ValueError(
"PICRUST currently only supports confidence intervals with the ace_ml and ace_reml ASR methods"
)
if opts.verbose:
print "Reconstruction method:", asr_method
print "Prediction method:", predict_traits_method
print "Parallel method:", parallel_method
print "num_jobs:", opts.num_jobs
print "\nOutput will be saved here:'%s'" % output_dir
#create the output directory unless it already exists
make_output_dir(output_dir)
if (parallel_method == 'sge'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs_sge.py')
elif (parallel_method == 'multithreaded'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs.py')
elif (parallel_method == 'torque'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs_torque.py')
else:
raise RuntimeError
#get the test datasets to run in the input directory (based on exp_traits files)
expect_test_files = glob(join(input_dir, 'exp_traits--*'))
test_datasets = {}
for file_name in expect_test_files:
test_id = file_name.replace(join(input_dir, 'exp_traits--'), '', 1)
#create a dict with the test files as values in the ref list
test_datasets[test_id] = [
join(input_dir, 'test_trait_table--' + test_id),
join(input_dir, 'test_tree--' + test_id),
join(input_dir, 'exp_traits--' + test_id)
]
created_tmp_files = []
output_files = []
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp = get_tmp_filename(tmp_dir=tmp_dir, prefix='jobs_')
jobs = open(jobs_fp, 'w')
created_tmp_files.append(jobs_fp)
#get location of scripts we need to run
asr_script_fp = join(get_picrust_project_dir(), 'scripts',
'ancestral_state_reconstruction.py')
predict_traits_script_fp = join(get_picrust_project_dir(), 'scripts',
'predict_traits.py')
#run each test dataset through the pipeline
for test_id in test_datasets:
asr_out_fp = join(output_dir, 'asr--' + asr_method + '--' + test_id)
asr_params_out_fp = join(
output_dir, '--'.join(['asr', asr_method, 'asr_params', test_id]))
created_tmp_files.append(asr_out_fp)
if opts.check_for_null_files and exists(
asr_out_fp) and file_contains_nulls(asr_out_fp):
#remove file
if opts.verbose:
print "Existing ASR file contains null characters. Will run ASR again after removing: " + asr_out_fp
remove(asr_out_fp)
if exists(asr_out_fp) and not opts.force:
if opts.verbose:
print "Output file: {0} already exists, so we will skip it.".format(
asr_out_fp)
asr_cmd = "echo 'Skipping ASR for %s, file %s exists already'" % (
test_id, asr_out_fp)
else:
#create the asr command
asr_cmd = """python {0} -i "{1}" -t "{2}" -m {3} -o "{4}" -c "{5}" """.format(
asr_script_fp, test_datasets[test_id][0],
test_datasets[test_id][1], asr_method, asr_out_fp,
asr_params_out_fp)
predict_traits_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\
opts.weighting_method,test_id]))
if opts.with_accuracy:
predict_traits_accuracy_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\
opts.weighting_method,'accuracy_metrics',test_id]))
if opts.check_for_null_files and exists(
predict_traits_out_fp) and file_contains_nulls(
predict_traits_out_fp):
if opts.verbose:
print "Existing trait predictions file contains null characters. Will run it again after removing: " + predict_traits_out_fp
remove(predict_traits_out_fp)
if exists(predict_traits_out_fp) and not opts.force:
if opts.verbose:
print "Prediction file: {0} already exists. Skipping ASR and prediction for this organism".format(
predict_traits_out_fp)
continue
output_files.append(predict_traits_out_fp)
genome_id = split('--', test_id)[2]
if predict_traits_method == 'nearest_neighbor':
#don't do asr step
predict_traits_cmd = """python {0} -i "{1}" -t "{2}" -g "{3}" -o "{4}" -m "{5}" """.format(
predict_traits_script_fp, test_datasets[test_id][0],
opts.ref_tree, genome_id, predict_traits_out_fp,
predict_traits_method)
jobs.write(predict_traits_cmd + "\n")
else:
#create the predict traits command
predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -r "{3}" -g "{4}" -o "{5}" -m "{6}" -w {7} """.format(predict_traits_script_fp,\
test_datasets[test_id][0], opts.ref_tree, asr_out_fp,genome_id, predict_traits_out_fp,predict_traits_method,opts.weighting_method)
#Instruct predict_traits to use confidence intervals output by ASR
if opts.with_confidence:
confidence_param = ' -c "%s"' % (asr_params_out_fp)
predict_traits_cmd = predict_traits_cmd + confidence_param
#Instruct predict traits to output the NTSI measure of distance to
#nearby sequences.
if opts.with_accuracy:
accuracy_param = ' -a "%s"' % (predict_traits_accuracy_out_fp)
predict_traits_cmd = predict_traits_cmd + accuracy_param
#add job command to the the jobs file
jobs.write(asr_cmd + ';' + predict_traits_cmd + "\n")
jobs.close()
#created_tmp_files.extend(output_files)
#submit the jobs
job_prefix = 'eval_'
if opts.verbose:
print "Submitting jobs:", cluster_jobs_fp, jobs_fp, job_prefix, opts.num_jobs
submit_jobs(cluster_jobs_fp, jobs_fp, job_prefix, num_jobs=opts.num_jobs)
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
#set some defaults for the options
input_dir=opts.input_dir
output_dir=opts.output_dir or input_dir
tmp_dir=opts.tmp_dir or output_dir
parallel_method=opts.parallel_method
asr_method = opts.asr_method
predict_traits_method = opts.prediction_method
if opts.num_jobs > 20 and parallel_method == 'multithreaded':
raise ValueError('You probably dont want to run multithreaded evaluations with a large num_jobs. Please adjust options num_jobs and or parallel_method')
if opts.with_confidence and asr_method not in ['ace_ml','ace_reml']:
raise ValueError("PICRUST currently only supports confidence intervals with the ace_ml and ace_reml ASR methods")
if opts.verbose:
print "Reconstruction method:",asr_method
print "Prediction method:",predict_traits_method
print "Parallel method:",parallel_method
print "num_jobs:",opts.num_jobs
print "\nOutput will be saved here:'%s'" %output_dir
#create the output directory unless it already exists
make_output_dir(output_dir)
if(parallel_method=='sge'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py')
elif(parallel_method=='multithreaded'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py')
elif(parallel_method=='torque'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py')
else:
raise RuntimeError
#get the test datasets to run in the input directory (based on exp_traits files)
expect_test_files=glob(join(input_dir,'exp_traits--*'))
test_datasets={}
for file_name in expect_test_files:
test_id=file_name.replace(join(input_dir,'exp_traits--'),'',1)
#create a dict with the test files as values in the ref list
test_datasets[test_id]=[ join(input_dir,'test_trait_table--'+test_id),join(input_dir,'test_tree--'+test_id),join(input_dir,'exp_traits--'+test_id)]
created_tmp_files=[]
output_files=[]
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_')
jobs=open(jobs_fp,'w')
created_tmp_files.append(jobs_fp)
#get location of scripts we need to run
asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py')
predict_traits_script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py')
#run each test dataset through the pipeline
for test_id in test_datasets:
asr_out_fp=join(output_dir,'asr--'+asr_method+'--'+test_id)
asr_params_out_fp=join(output_dir,'--'.join(['asr',asr_method,'asr_params',test_id]))
created_tmp_files.append(asr_out_fp)
if opts.check_for_null_files and exists(asr_out_fp) and file_contains_nulls(asr_out_fp):
#remove file
if opts.verbose:
print "Existing ASR file contains null characters. Will run ASR again after removing: "+asr_out_fp
remove(asr_out_fp)
if exists(asr_out_fp) and not opts.force:
if opts.verbose:
print "Output file: {0} already exists, so we will skip it.".format(asr_out_fp)
asr_cmd = "echo 'Skipping ASR for %s, file %s exists already'" %(test_id,asr_out_fp)
else:
#create the asr command
asr_cmd= """python {0} -i "{1}" -t "{2}" -m {3} -o "{4}" -c "{5}" """.format(asr_script_fp, test_datasets[test_id][0], test_datasets[test_id][1], asr_method, asr_out_fp, asr_params_out_fp)
predict_traits_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\
opts.weighting_method,test_id]))
if opts.with_accuracy:
predict_traits_accuracy_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\
opts.weighting_method,'accuracy_metrics',test_id]))
if opts.check_for_null_files and exists(predict_traits_out_fp) and file_contains_nulls(predict_traits_out_fp):
if opts.verbose:
print "Existing trait predictions file contains null characters. Will run it again after removing: "+predict_traits_out_fp
remove(predict_traits_out_fp)
if exists(predict_traits_out_fp) and not opts.force:
if opts.verbose:
print "Prediction file: {0} already exists. Skipping ASR and prediction for this organism".format(predict_traits_out_fp)
continue
output_files.append(predict_traits_out_fp)
genome_id=split('--',test_id)[2]
if predict_traits_method == 'nearest_neighbor':
#don't do asr step
predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -g "{3}" -o "{4}" -m "{5}" """.format(predict_traits_script_fp, test_datasets[test_id][0], opts.ref_tree, genome_id, predict_traits_out_fp,predict_traits_method)
jobs.write(predict_traits_cmd+"\n")
else:
#create the predict traits command
predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -r "{3}" -g "{4}" -o "{5}" -m "{6}" -w {7} """.format(predict_traits_script_fp,\
test_datasets[test_id][0], opts.ref_tree, asr_out_fp,genome_id, predict_traits_out_fp,predict_traits_method,opts.weighting_method)
#Instruct predict_traits to use confidence intervals output by ASR
if opts.with_confidence:
confidence_param = ' -c "%s"' %(asr_params_out_fp)
predict_traits_cmd = predict_traits_cmd + confidence_param
#Instruct predict traits to output the NTSI measure of distance to
#nearby sequences.
if opts.with_accuracy:
accuracy_param = ' -a "%s"' %(predict_traits_accuracy_out_fp)
predict_traits_cmd = predict_traits_cmd + accuracy_param
#add job command to the the jobs file
jobs.write(asr_cmd+';'+predict_traits_cmd+"\n")
jobs.close()
#created_tmp_files.extend(output_files)
#submit the jobs
job_prefix='eval_'
if opts.verbose:
print "Submitting jobs:",cluster_jobs_fp,jobs_fp,job_prefix,opts.num_jobs
submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=opts.num_jobs)
def run_asr_in_parallel(tree, table, asr_method, parallel_method='sge',tmp_dir='jobs/',num_jobs=100, verbose=False):
'''Runs the ancestral state reconstructions in parallel'''
asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py')
if(parallel_method=='sge'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs_sge.py')
elif(parallel_method=='multithreaded'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs.py')
elif(parallel_method=='torque'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs_torque.py')
else:
raise RuntimeError
if(verbose):
print "Loading trait table..."
#foreach trait in the table, create a new tmp file with just that trait, and create the job command and add it a tmp jobs file
table=LoadTable(filename=table, header=True, sep='\t')
#get dimensions of the table
dim=table.Shape
created_tmp_files=[]
output_files=[]
ci_files=[]
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_asr_')
jobs=open(jobs_fp,'w')
created_tmp_files.append(jobs_fp)
if(verbose):
print "Creating temporary input files in: ",tmp_dir
#iterate over each column
for i in range(1,dim[1]):
#create a new table with only a single trait
single_col_table=table.getColumns([0,i])
#write the new table to a tmp file
single_col_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='in_asr_')
single_col_table.writeToFile(single_col_fp,sep='\t')
created_tmp_files.append(single_col_fp)
#create tmp output files
tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_')
output_files.append(tmp_output_fp)
tmp_ci_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_ci_')
ci_files.append(tmp_ci_fp)
#create the job command
cmd= "{0} -i {1} -t {2} -m {3} -o {4} -c {5}".format(asr_script_fp, single_col_fp, tree, asr_method, tmp_output_fp, tmp_ci_fp)
#add job command to the the jobs file
jobs.write(cmd+"\n")
jobs.close()
created_tmp_files.extend(output_files)
created_tmp_files.extend(ci_files)
if(verbose):
print "Launching parallel jobs."
#run the job command
job_prefix='asr'
submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=num_jobs)
if(verbose):
print "Jobs are now running. Will wait until finished."
#wait until all jobs finished (e.g. simple poller)
wait_for_output_files(output_files)
if(verbose):
print "Jobs are done running. Now combining all tmp files."
#Combine output files
combined_table=combine_asr_tables(output_files)
combined_ci_table=combine_asr_tables(ci_files)
#create a Table object
combined_table=Table(header=combined_table[0],rows=combined_table[1:])
combined_ci_table=Table(header=combined_ci_table[0],rows=combined_ci_table[1:])
#clean up all tmp files
for file in created_tmp_files:
remove(file)
#return the combined table
return combined_table,combined_ci_table
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
tmp_dir='jobs/'
make_output_dir(tmp_dir)
#Run the jobs
script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py')
if(opts.parallel_method=='sge'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py')
elif(opts.parallel_method=='multithreaded'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py')
elif(opts.parallel_method=='torque'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py')
else:
raise RuntimeError
if(opts.verbose):
print "Loading tree..."
tree = load_picrust_tree(opts.tree, opts.verbose)
all_tips = [tip.Name for tip in tree.tips()]
if(opts.verbose):
print "Total number of possible tips to predict: {0}".format(len(all_tips))
created_tmp_files=[]
output_files={}
output_files['counts']=[]
if opts.reconstruction_confidence:
output_files['variances']=[]
output_files['upper_CI']=[]
output_files['lower_CI']=[]
if opts.already_calculated:
all_tips=get_tips_not_in_precalc(all_tips,opts.already_calculated)
if opts.verbose:
print "After taking into account tips already predicted, the number of tips left to predict is: {0}".format(len(all_tips))
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_')
jobs=open(jobs_fp,'w')
created_tmp_files.append(jobs_fp)
if(opts.verbose):
print "Creating temporary input files in: ",tmp_dir
num_tips_per_job=1000
for tips_to_predict in [all_tips[i:i+num_tips_per_job] for i in range(0, len(all_tips), num_tips_per_job)]:
#create tmp output files
tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_predict_traits_')
output_files['counts'].append(tmp_output_fp)
tip_to_predict_str=','.join(list(tips_to_predict))
if opts.reconstruction_confidence:
outfile_base,extension = splitext(tmp_output_fp)
output_files['variances'].append(outfile_base+"_variances.tab")
output_files['upper_CI'].append(outfile_base+"_upper_CI.tab")
output_files['lower_CI'].append(outfile_base+"_lower_CI.tab")
#create the job command
cmd= "{0} -i {1} -t {2} -r {3} -c {4} -g {5} -o {6}".format(script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, opts.reconstruction_confidence, tip_to_predict_str, tmp_output_fp)
else:
cmd= "{0} -i {1} -t {2} -r {3} -g {4} -o {5}".format(script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, tip_to_predict_str, tmp_output_fp)
#NOTE: Calculating NSTI this way is convenient,
#but would probably be faster if we ran the NSTI calculation separate (using the --output_accuracy_metrics_only) and added it to the output file later on.
if opts.calculate_accuracy_metrics:
cmd=cmd+" -a"
#add job command to the the jobs file
jobs.write(cmd+"\n")
jobs.close()
#add all output files to tmp list (used later for deletion)
for predict_type in output_files:
created_tmp_files.extend(output_files[predict_type])
if(opts.verbose):
print "Launching parallel jobs."
#run the job command
job_prefix='picrust'
submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=opts.num_jobs,delay=opts.delay)
if(opts.verbose):
print "Jobs are now running. Will wait until finished."
#wait until all jobs finished (e.g. simple poller)
wait_for_output_files(output_files['counts'])
if(opts.verbose):
print "Jobs are done running."
make_output_dir_for_file(opts.output_trait_table)
outfile_base,extension = splitext(opts.output_trait_table)
for predict_type in sorted(output_files):
#Combine output files
if opts.verbose:
print "Combining all output files for "+ predict_type
combined_predictions=combine_predict_trait_output(output_files[predict_type])
if opts.verbose:
print "Writing combined file for "+predict_type
if predict_type == 'counts':
#Output in whatever format the user wants
if opts.output_precalc_file_in_biom:
open(opts.output_trait_table,'w').write(format_biom_table(convert_precalc_to_biom(combined_predictions)))
else:
open(opts.output_trait_table,'w').write(combined_predictions)
else:
if opts.output_precalc_file_in_biom:
open(outfile_base+"_"+predict_type+".biom",'w').write(format_biom_table(convert_precalc_to_biom(combined_predictions)))
else:
open(outfile_base+"_"+predict_type+".tab",'w').write(combined_predictions)
#clean up all tmp files
for file in created_tmp_files:
remove(file)
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
tmp_dir = 'jobs/'
make_output_dir(tmp_dir)
#Run the jobs
script_fp = join(get_picrust_project_dir(), 'scripts', 'predict_traits.py')
if (opts.parallel_method == 'sge'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs_sge.py')
elif (opts.parallel_method == 'multithreaded'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs.py')
elif (opts.parallel_method == 'torque'):
cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts',
'start_parallel_jobs_torque.py')
else:
raise RuntimeError
if (opts.verbose):
print "Loading tree..."
tree = load_picrust_tree(opts.tree, opts.verbose)
all_tips = [tip.Name for tip in tree.tips()]
if (opts.verbose):
print "Total number of possible tips to predict: {0}".format(
len(all_tips))
created_tmp_files = []
output_files = {}
output_files['counts'] = []
if opts.reconstruction_confidence:
output_files['variances'] = []
output_files['upper_CI'] = []
output_files['lower_CI'] = []
if opts.already_calculated:
all_tips = get_tips_not_in_precalc(all_tips, opts.already_calculated)
if opts.verbose:
print "After taking into account tips already predicted, the number of tips left to predict is: {0}".format(
len(all_tips))
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp = get_tmp_filename(tmp_dir=tmp_dir, prefix='jobs_')
jobs = open(jobs_fp, 'w')
created_tmp_files.append(jobs_fp)
if (opts.verbose):
print "Creating temporary input files in: ", tmp_dir
num_tips_per_job = 1000
for tips_to_predict in [
all_tips[i:i + num_tips_per_job]
for i in range(0, len(all_tips), num_tips_per_job)
]:
#create tmp output files
tmp_output_fp = get_tmp_filename(tmp_dir=tmp_dir,
prefix='out_predict_traits_')
output_files['counts'].append(tmp_output_fp)
tip_to_predict_str = ','.join(list(tips_to_predict))
if opts.reconstruction_confidence:
outfile_base, extension = splitext(tmp_output_fp)
output_files['variances'].append(outfile_base + "_variances.tab")
output_files['upper_CI'].append(outfile_base + "_upper_CI.tab")
output_files['lower_CI'].append(outfile_base + "_lower_CI.tab")
#create the job command
cmd = "{0} -i {1} -t {2} -r {3} -c {4} -g {5} -o {6}".format(
script_fp, opts.observed_trait_table, opts.tree,
opts.reconstructed_trait_table, opts.reconstruction_confidence,
tip_to_predict_str, tmp_output_fp)
else:
cmd = "{0} -i {1} -t {2} -r {3} -g {4} -o {5}".format(
script_fp, opts.observed_trait_table, opts.tree,
opts.reconstructed_trait_table, tip_to_predict_str,
tmp_output_fp)
#NOTE: Calculating NSTI this way is convenient,
#but would probably be faster if we ran the NSTI calculation separate (using the --output_accuracy_metrics_only) and added it to the output file later on.
if opts.calculate_accuracy_metrics:
cmd = cmd + " -a"
#add job command to the the jobs file
jobs.write(cmd + "\n")
jobs.close()
#add all output files to tmp list (used later for deletion)
for predict_type in output_files:
created_tmp_files.extend(output_files[predict_type])
if (opts.verbose):
print "Launching parallel jobs."
#run the job command
job_prefix = 'picrust'
submit_jobs(cluster_jobs_fp,
jobs_fp,
job_prefix,
num_jobs=opts.num_jobs,
delay=opts.delay)
if (opts.verbose):
print "Jobs are now running. Will wait until finished."
#wait until all jobs finished (e.g. simple poller)
wait_for_output_files(output_files['counts'])
if (opts.verbose):
print "Jobs are done running."
make_output_dir_for_file(opts.output_trait_table)
outfile_base, extension = splitext(opts.output_trait_table)
for predict_type in sorted(output_files):
#Combine output files
if opts.verbose:
print "Combining all output files for " + predict_type
combined_predictions = combine_predict_trait_output(
output_files[predict_type])
if opts.verbose:
print "Writing combined file for " + predict_type
if predict_type == 'counts':
#Output in whatever format the user wants
if opts.output_precalc_file_in_biom:
open(opts.output_trait_table, 'w').write(
format_biom_table(
convert_precalc_to_biom(combined_predictions)))
else:
open(opts.output_trait_table, 'w').write(combined_predictions)
else:
if opts.output_precalc_file_in_biom:
open(outfile_base + "_" + predict_type + ".biom", 'w').write(
format_biom_table(
convert_precalc_to_biom(combined_predictions)))
else:
open(outfile_base + "_" + predict_type + ".tab",
'w').write(combined_predictions)
#clean up all tmp files
for file in created_tmp_files:
remove(file)
def run_asr_in_parallel(tree, table, asr_method, parallel_method='sge',tmp_dir='jobs/',num_jobs=100, verbose=False):
'''Runs the ancestral state reconstructions in parallel'''
asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py')
if(parallel_method=='sge'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py')
elif(parallel_method=='multithreaded'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py')
elif(parallel_method=='torque'):
cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py')
else:
raise RuntimeError
if(verbose):
print "Loading trait table..."
#foreach trait in the table, create a new tmp file with just that trait, and create the job command and add it a tmp jobs file
table=LoadTable(filename=table, header=True, sep='\t')
#get dimensions of the table
dim=table.Shape
created_tmp_files=[]
output_files=[]
ci_files=[]
#create a tmp file to store the job commands (which we will pass to our parallel script to run)
jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_asr_')
jobs=open(jobs_fp,'w')
created_tmp_files.append(jobs_fp)
if(verbose):
print "Creating temporary input files in: ",tmp_dir
#iterate over each column
for i in range(1,dim[1]):
#create a new table with only a single trait
single_col_table=table.getColumns([0,i])
#write the new table to a tmp file
single_col_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='in_asr_')
single_col_table.writeToFile(single_col_fp,sep='\t')
created_tmp_files.append(single_col_fp)
#create tmp output files
tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_')
output_files.append(tmp_output_fp)
tmp_ci_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_ci_')
ci_files.append(tmp_ci_fp)
#create the job command
cmd= "{0} -i {1} -t {2} -m {3} -o {4} -c {5}".format(asr_script_fp, single_col_fp, tree, asr_method, tmp_output_fp, tmp_ci_fp)
#add job command to the the jobs file
jobs.write(cmd+"\n")
jobs.close()
created_tmp_files.extend(output_files)
created_tmp_files.extend(ci_files)
if(verbose):
print "Launching parallel jobs."
#run the job command
job_prefix='asr'
submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=num_jobs)
if(verbose):
print "Jobs are now running. Will wait until finished."
#wait until all jobs finished (e.g. simple poller)
wait_for_output_files(output_files)
if(verbose):
print "Jobs are done running. Now combining all tmp files."
#Combine output files
combined_table=combine_asr_tables(output_files)
combined_ci_table=combine_asr_tables(ci_files)
#create a Table object
combined_table=Table(header=combined_table[0],rows=combined_table[1:])
combined_ci_table=Table(header=combined_ci_table[0],rows=combined_ci_table[1:])
#clean up all tmp files
for file in created_tmp_files:
remove(file)
#return the combined table
return combined_table,combined_ci_table
