def main(): option_parser, opts, args =\ parse_command_line_parameters(**script_info) #set some defaults for the options input_dir = opts.input_dir output_dir = opts.output_dir or input_dir tmp_dir = opts.tmp_dir or output_dir parallel_method = opts.parallel_method asr_method = opts.asr_method predict_traits_method = opts.prediction_method if opts.num_jobs > 20 and parallel_method == 'multithreaded': raise ValueError( 'You probably dont want to run multithreaded evaluations with a large num_jobs. Please adjust options num_jobs and or parallel_method' ) if opts.with_confidence and asr_method not in ['ace_ml', 'ace_reml']: raise ValueError( "PICRUST currently only supports confidence intervals with the ace_ml and ace_reml ASR methods" ) if opts.verbose: print "Reconstruction method:", asr_method print "Prediction method:", predict_traits_method print "Parallel method:", parallel_method print "num_jobs:", opts.num_jobs print "\nOutput will be saved here:'%s'" % output_dir #create the output directory unless it already exists make_output_dir(output_dir) if (parallel_method == 'sge'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs_sge.py') elif (parallel_method == 'multithreaded'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs.py') elif (parallel_method == 'torque'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs_torque.py') else: raise RuntimeError #get the test datasets to run in the input directory (based on exp_traits files) expect_test_files = glob(join(input_dir, 'exp_traits--*')) test_datasets = {} for file_name in expect_test_files: test_id = file_name.replace(join(input_dir, 'exp_traits--'), '', 1) #create a dict with the test files as values in the ref list test_datasets[test_id] = [ join(input_dir, 'test_trait_table--' + test_id), join(input_dir, 'test_tree--' + test_id), join(input_dir, 'exp_traits--' + test_id) ] created_tmp_files = [] output_files = [] #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp = get_tmp_filename(tmp_dir=tmp_dir, prefix='jobs_') jobs = open(jobs_fp, 'w') created_tmp_files.append(jobs_fp) #get location of scripts we need to run asr_script_fp = join(get_picrust_project_dir(), 'scripts', 'ancestral_state_reconstruction.py') predict_traits_script_fp = join(get_picrust_project_dir(), 'scripts', 'predict_traits.py') #run each test dataset through the pipeline for test_id in test_datasets: asr_out_fp = join(output_dir, 'asr--' + asr_method + '--' + test_id) asr_params_out_fp = join( output_dir, '--'.join(['asr', asr_method, 'asr_params', test_id])) created_tmp_files.append(asr_out_fp) if opts.check_for_null_files and exists( asr_out_fp) and file_contains_nulls(asr_out_fp): #remove file if opts.verbose: print "Existing ASR file contains null characters. Will run ASR again after removing: " + asr_out_fp remove(asr_out_fp) if exists(asr_out_fp) and not opts.force: if opts.verbose: print "Output file: {0} already exists, so we will skip it.".format( asr_out_fp) asr_cmd = "echo 'Skipping ASR for %s, file %s exists already'" % ( test_id, asr_out_fp) else: #create the asr command asr_cmd = """python {0} -i "{1}" -t "{2}" -m {3} -o "{4}" -c "{5}" """.format( asr_script_fp, test_datasets[test_id][0], test_datasets[test_id][1], asr_method, asr_out_fp, asr_params_out_fp) predict_traits_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\ opts.weighting_method,test_id])) if opts.with_accuracy: predict_traits_accuracy_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\ opts.weighting_method,'accuracy_metrics',test_id])) if opts.check_for_null_files and exists( predict_traits_out_fp) and file_contains_nulls( predict_traits_out_fp): if opts.verbose: print "Existing trait predictions file contains null characters. Will run it again after removing: " + predict_traits_out_fp remove(predict_traits_out_fp) if exists(predict_traits_out_fp) and not opts.force: if opts.verbose: print "Prediction file: {0} already exists. Skipping ASR and prediction for this organism".format( predict_traits_out_fp) continue output_files.append(predict_traits_out_fp) genome_id = split('--', test_id)[2] if predict_traits_method == 'nearest_neighbor': #don't do asr step predict_traits_cmd = """python {0} -i "{1}" -t "{2}" -g "{3}" -o "{4}" -m "{5}" """.format( predict_traits_script_fp, test_datasets[test_id][0], opts.ref_tree, genome_id, predict_traits_out_fp, predict_traits_method) jobs.write(predict_traits_cmd + "\n") else: #create the predict traits command predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -r "{3}" -g "{4}" -o "{5}" -m "{6}" -w {7} """.format(predict_traits_script_fp,\ test_datasets[test_id][0], opts.ref_tree, asr_out_fp,genome_id, predict_traits_out_fp,predict_traits_method,opts.weighting_method) #Instruct predict_traits to use confidence intervals output by ASR if opts.with_confidence: confidence_param = ' -c "%s"' % (asr_params_out_fp) predict_traits_cmd = predict_traits_cmd + confidence_param #Instruct predict traits to output the NTSI measure of distance to #nearby sequences. if opts.with_accuracy: accuracy_param = ' -a "%s"' % (predict_traits_accuracy_out_fp) predict_traits_cmd = predict_traits_cmd + accuracy_param #add job command to the the jobs file jobs.write(asr_cmd + ';' + predict_traits_cmd + "\n") jobs.close() #created_tmp_files.extend(output_files) #submit the jobs job_prefix = 'eval_' if opts.verbose: print "Submitting jobs:", cluster_jobs_fp, jobs_fp, job_prefix, opts.num_jobs submit_jobs(cluster_jobs_fp, jobs_fp, job_prefix, num_jobs=opts.num_jobs)
def main(): option_parser, opts, args =\ parse_command_line_parameters(**script_info) #set some defaults for the options input_dir=opts.input_dir output_dir=opts.output_dir or input_dir tmp_dir=opts.tmp_dir or output_dir parallel_method=opts.parallel_method asr_method = opts.asr_method predict_traits_method = opts.prediction_method if opts.num_jobs > 20 and parallel_method == 'multithreaded': raise ValueError('You probably dont want to run multithreaded evaluations with a large num_jobs. Please adjust options num_jobs and or parallel_method') if opts.with_confidence and asr_method not in ['ace_ml','ace_reml']: raise ValueError("PICRUST currently only supports confidence intervals with the ace_ml and ace_reml ASR methods") if opts.verbose: print "Reconstruction method:",asr_method print "Prediction method:",predict_traits_method print "Parallel method:",parallel_method print "num_jobs:",opts.num_jobs print "\nOutput will be saved here:'%s'" %output_dir #create the output directory unless it already exists make_output_dir(output_dir) if(parallel_method=='sge'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py') elif(parallel_method=='multithreaded'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py') elif(parallel_method=='torque'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py') else: raise RuntimeError #get the test datasets to run in the input directory (based on exp_traits files) expect_test_files=glob(join(input_dir,'exp_traits--*')) test_datasets={} for file_name in expect_test_files: test_id=file_name.replace(join(input_dir,'exp_traits--'),'',1) #create a dict with the test files as values in the ref list test_datasets[test_id]=[ join(input_dir,'test_trait_table--'+test_id),join(input_dir,'test_tree--'+test_id),join(input_dir,'exp_traits--'+test_id)] created_tmp_files=[] output_files=[] #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_') jobs=open(jobs_fp,'w') created_tmp_files.append(jobs_fp) #get location of scripts we need to run asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py') predict_traits_script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py') #run each test dataset through the pipeline for test_id in test_datasets: asr_out_fp=join(output_dir,'asr--'+asr_method+'--'+test_id) asr_params_out_fp=join(output_dir,'--'.join(['asr',asr_method,'asr_params',test_id])) created_tmp_files.append(asr_out_fp) if opts.check_for_null_files and exists(asr_out_fp) and file_contains_nulls(asr_out_fp): #remove file if opts.verbose: print "Existing ASR file contains null characters. Will run ASR again after removing: "+asr_out_fp remove(asr_out_fp) if exists(asr_out_fp) and not opts.force: if opts.verbose: print "Output file: {0} already exists, so we will skip it.".format(asr_out_fp) asr_cmd = "echo 'Skipping ASR for %s, file %s exists already'" %(test_id,asr_out_fp) else: #create the asr command asr_cmd= """python {0} -i "{1}" -t "{2}" -m {3} -o "{4}" -c "{5}" """.format(asr_script_fp, test_datasets[test_id][0], test_datasets[test_id][1], asr_method, asr_out_fp, asr_params_out_fp) predict_traits_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\ opts.weighting_method,test_id])) if opts.with_accuracy: predict_traits_accuracy_out_fp=join(output_dir,'--'.join(['predict_traits',predict_traits_method,\ opts.weighting_method,'accuracy_metrics',test_id])) if opts.check_for_null_files and exists(predict_traits_out_fp) and file_contains_nulls(predict_traits_out_fp): if opts.verbose: print "Existing trait predictions file contains null characters. Will run it again after removing: "+predict_traits_out_fp remove(predict_traits_out_fp) if exists(predict_traits_out_fp) and not opts.force: if opts.verbose: print "Prediction file: {0} already exists. Skipping ASR and prediction for this organism".format(predict_traits_out_fp) continue output_files.append(predict_traits_out_fp) genome_id=split('--',test_id)[2] if predict_traits_method == 'nearest_neighbor': #don't do asr step predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -g "{3}" -o "{4}" -m "{5}" """.format(predict_traits_script_fp, test_datasets[test_id][0], opts.ref_tree, genome_id, predict_traits_out_fp,predict_traits_method) jobs.write(predict_traits_cmd+"\n") else: #create the predict traits command predict_traits_cmd= """python {0} -i "{1}" -t "{2}" -r "{3}" -g "{4}" -o "{5}" -m "{6}" -w {7} """.format(predict_traits_script_fp,\ test_datasets[test_id][0], opts.ref_tree, asr_out_fp,genome_id, predict_traits_out_fp,predict_traits_method,opts.weighting_method) #Instruct predict_traits to use confidence intervals output by ASR if opts.with_confidence: confidence_param = ' -c "%s"' %(asr_params_out_fp) predict_traits_cmd = predict_traits_cmd + confidence_param #Instruct predict traits to output the NTSI measure of distance to #nearby sequences. if opts.with_accuracy: accuracy_param = ' -a "%s"' %(predict_traits_accuracy_out_fp) predict_traits_cmd = predict_traits_cmd + accuracy_param #add job command to the the jobs file jobs.write(asr_cmd+';'+predict_traits_cmd+"\n") jobs.close() #created_tmp_files.extend(output_files) #submit the jobs job_prefix='eval_' if opts.verbose: print "Submitting jobs:",cluster_jobs_fp,jobs_fp,job_prefix,opts.num_jobs submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=opts.num_jobs)
def run_asr_in_parallel(tree, table, asr_method, parallel_method='sge',tmp_dir='jobs/',num_jobs=100, verbose=False): '''Runs the ancestral state reconstructions in parallel''' asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py') if(parallel_method=='sge'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs_sge.py') elif(parallel_method=='multithreaded'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs.py') elif(parallel_method=='torque'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_picrust_jobs_torque.py') else: raise RuntimeError if(verbose): print "Loading trait table..." #foreach trait in the table, create a new tmp file with just that trait, and create the job command and add it a tmp jobs file table=LoadTable(filename=table, header=True, sep='\t') #get dimensions of the table dim=table.Shape created_tmp_files=[] output_files=[] ci_files=[] #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_asr_') jobs=open(jobs_fp,'w') created_tmp_files.append(jobs_fp) if(verbose): print "Creating temporary input files in: ",tmp_dir #iterate over each column for i in range(1,dim[1]): #create a new table with only a single trait single_col_table=table.getColumns([0,i]) #write the new table to a tmp file single_col_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='in_asr_') single_col_table.writeToFile(single_col_fp,sep='\t') created_tmp_files.append(single_col_fp) #create tmp output files tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_') output_files.append(tmp_output_fp) tmp_ci_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_ci_') ci_files.append(tmp_ci_fp) #create the job command cmd= "{0} -i {1} -t {2} -m {3} -o {4} -c {5}".format(asr_script_fp, single_col_fp, tree, asr_method, tmp_output_fp, tmp_ci_fp) #add job command to the the jobs file jobs.write(cmd+"\n") jobs.close() created_tmp_files.extend(output_files) created_tmp_files.extend(ci_files) if(verbose): print "Launching parallel jobs." #run the job command job_prefix='asr' submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=num_jobs) if(verbose): print "Jobs are now running. Will wait until finished." #wait until all jobs finished (e.g. simple poller) wait_for_output_files(output_files) if(verbose): print "Jobs are done running. Now combining all tmp files." #Combine output files combined_table=combine_asr_tables(output_files) combined_ci_table=combine_asr_tables(ci_files) #create a Table object combined_table=Table(header=combined_table[0],rows=combined_table[1:]) combined_ci_table=Table(header=combined_ci_table[0],rows=combined_ci_table[1:]) #clean up all tmp files for file in created_tmp_files: remove(file) #return the combined table return combined_table,combined_ci_table
def main(): option_parser, opts, args =\ parse_command_line_parameters(**script_info) tmp_dir='jobs/' make_output_dir(tmp_dir) #Run the jobs script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py') if(opts.parallel_method=='sge'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py') elif(opts.parallel_method=='multithreaded'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py') elif(opts.parallel_method=='torque'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py') else: raise RuntimeError if(opts.verbose): print "Loading tree..." tree = load_picrust_tree(opts.tree, opts.verbose) all_tips = [tip.Name for tip in tree.tips()] if(opts.verbose): print "Total number of possible tips to predict: {0}".format(len(all_tips)) created_tmp_files=[] output_files={} output_files['counts']=[] if opts.reconstruction_confidence: output_files['variances']=[] output_files['upper_CI']=[] output_files['lower_CI']=[] if opts.already_calculated: all_tips=get_tips_not_in_precalc(all_tips,opts.already_calculated) if opts.verbose: print "After taking into account tips already predicted, the number of tips left to predict is: {0}".format(len(all_tips)) #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_') jobs=open(jobs_fp,'w') created_tmp_files.append(jobs_fp) if(opts.verbose): print "Creating temporary input files in: ",tmp_dir num_tips_per_job=1000 for tips_to_predict in [all_tips[i:i+num_tips_per_job] for i in range(0, len(all_tips), num_tips_per_job)]: #create tmp output files tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_predict_traits_') output_files['counts'].append(tmp_output_fp) tip_to_predict_str=','.join(list(tips_to_predict)) if opts.reconstruction_confidence: outfile_base,extension = splitext(tmp_output_fp) output_files['variances'].append(outfile_base+"_variances.tab") output_files['upper_CI'].append(outfile_base+"_upper_CI.tab") output_files['lower_CI'].append(outfile_base+"_lower_CI.tab") #create the job command cmd= "{0} -i {1} -t {2} -r {3} -c {4} -g {5} -o {6}".format(script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, opts.reconstruction_confidence, tip_to_predict_str, tmp_output_fp) else: cmd= "{0} -i {1} -t {2} -r {3} -g {4} -o {5}".format(script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, tip_to_predict_str, tmp_output_fp) #NOTE: Calculating NSTI this way is convenient, #but would probably be faster if we ran the NSTI calculation separate (using the --output_accuracy_metrics_only) and added it to the output file later on. if opts.calculate_accuracy_metrics: cmd=cmd+" -a" #add job command to the the jobs file jobs.write(cmd+"\n") jobs.close() #add all output files to tmp list (used later for deletion) for predict_type in output_files: created_tmp_files.extend(output_files[predict_type]) if(opts.verbose): print "Launching parallel jobs." #run the job command job_prefix='picrust' submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=opts.num_jobs,delay=opts.delay) if(opts.verbose): print "Jobs are now running. Will wait until finished." #wait until all jobs finished (e.g. simple poller) wait_for_output_files(output_files['counts']) if(opts.verbose): print "Jobs are done running." make_output_dir_for_file(opts.output_trait_table) outfile_base,extension = splitext(opts.output_trait_table) for predict_type in sorted(output_files): #Combine output files if opts.verbose: print "Combining all output files for "+ predict_type combined_predictions=combine_predict_trait_output(output_files[predict_type]) if opts.verbose: print "Writing combined file for "+predict_type if predict_type == 'counts': #Output in whatever format the user wants if opts.output_precalc_file_in_biom: open(opts.output_trait_table,'w').write(format_biom_table(convert_precalc_to_biom(combined_predictions))) else: open(opts.output_trait_table,'w').write(combined_predictions) else: if opts.output_precalc_file_in_biom: open(outfile_base+"_"+predict_type+".biom",'w').write(format_biom_table(convert_precalc_to_biom(combined_predictions))) else: open(outfile_base+"_"+predict_type+".tab",'w').write(combined_predictions) #clean up all tmp files for file in created_tmp_files: remove(file)
def main(): option_parser, opts, args =\ parse_command_line_parameters(**script_info) tmp_dir = 'jobs/' make_output_dir(tmp_dir) #Run the jobs script_fp = join(get_picrust_project_dir(), 'scripts', 'predict_traits.py') if (opts.parallel_method == 'sge'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs_sge.py') elif (opts.parallel_method == 'multithreaded'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs.py') elif (opts.parallel_method == 'torque'): cluster_jobs_fp = join(get_picrust_project_dir(), 'scripts', 'start_parallel_jobs_torque.py') else: raise RuntimeError if (opts.verbose): print "Loading tree..." tree = load_picrust_tree(opts.tree, opts.verbose) all_tips = [tip.Name for tip in tree.tips()] if (opts.verbose): print "Total number of possible tips to predict: {0}".format( len(all_tips)) created_tmp_files = [] output_files = {} output_files['counts'] = [] if opts.reconstruction_confidence: output_files['variances'] = [] output_files['upper_CI'] = [] output_files['lower_CI'] = [] if opts.already_calculated: all_tips = get_tips_not_in_precalc(all_tips, opts.already_calculated) if opts.verbose: print "After taking into account tips already predicted, the number of tips left to predict is: {0}".format( len(all_tips)) #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp = get_tmp_filename(tmp_dir=tmp_dir, prefix='jobs_') jobs = open(jobs_fp, 'w') created_tmp_files.append(jobs_fp) if (opts.verbose): print "Creating temporary input files in: ", tmp_dir num_tips_per_job = 1000 for tips_to_predict in [ all_tips[i:i + num_tips_per_job] for i in range(0, len(all_tips), num_tips_per_job) ]: #create tmp output files tmp_output_fp = get_tmp_filename(tmp_dir=tmp_dir, prefix='out_predict_traits_') output_files['counts'].append(tmp_output_fp) tip_to_predict_str = ','.join(list(tips_to_predict)) if opts.reconstruction_confidence: outfile_base, extension = splitext(tmp_output_fp) output_files['variances'].append(outfile_base + "_variances.tab") output_files['upper_CI'].append(outfile_base + "_upper_CI.tab") output_files['lower_CI'].append(outfile_base + "_lower_CI.tab") #create the job command cmd = "{0} -i {1} -t {2} -r {3} -c {4} -g {5} -o {6}".format( script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, opts.reconstruction_confidence, tip_to_predict_str, tmp_output_fp) else: cmd = "{0} -i {1} -t {2} -r {3} -g {4} -o {5}".format( script_fp, opts.observed_trait_table, opts.tree, opts.reconstructed_trait_table, tip_to_predict_str, tmp_output_fp) #NOTE: Calculating NSTI this way is convenient, #but would probably be faster if we ran the NSTI calculation separate (using the --output_accuracy_metrics_only) and added it to the output file later on. if opts.calculate_accuracy_metrics: cmd = cmd + " -a" #add job command to the the jobs file jobs.write(cmd + "\n") jobs.close() #add all output files to tmp list (used later for deletion) for predict_type in output_files: created_tmp_files.extend(output_files[predict_type]) if (opts.verbose): print "Launching parallel jobs." #run the job command job_prefix = 'picrust' submit_jobs(cluster_jobs_fp, jobs_fp, job_prefix, num_jobs=opts.num_jobs, delay=opts.delay) if (opts.verbose): print "Jobs are now running. Will wait until finished." #wait until all jobs finished (e.g. simple poller) wait_for_output_files(output_files['counts']) if (opts.verbose): print "Jobs are done running." make_output_dir_for_file(opts.output_trait_table) outfile_base, extension = splitext(opts.output_trait_table) for predict_type in sorted(output_files): #Combine output files if opts.verbose: print "Combining all output files for " + predict_type combined_predictions = combine_predict_trait_output( output_files[predict_type]) if opts.verbose: print "Writing combined file for " + predict_type if predict_type == 'counts': #Output in whatever format the user wants if opts.output_precalc_file_in_biom: open(opts.output_trait_table, 'w').write( format_biom_table( convert_precalc_to_biom(combined_predictions))) else: open(opts.output_trait_table, 'w').write(combined_predictions) else: if opts.output_precalc_file_in_biom: open(outfile_base + "_" + predict_type + ".biom", 'w').write( format_biom_table( convert_precalc_to_biom(combined_predictions))) else: open(outfile_base + "_" + predict_type + ".tab", 'w').write(combined_predictions) #clean up all tmp files for file in created_tmp_files: remove(file)
def run_asr_in_parallel(tree, table, asr_method, parallel_method='sge',tmp_dir='jobs/',num_jobs=100, verbose=False): '''Runs the ancestral state reconstructions in parallel''' asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py') if(parallel_method=='sge'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_sge.py') elif(parallel_method=='multithreaded'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs.py') elif(parallel_method=='torque'): cluster_jobs_fp=join(get_picrust_project_dir(),'scripts','start_parallel_jobs_torque.py') else: raise RuntimeError if(verbose): print "Loading trait table..." #foreach trait in the table, create a new tmp file with just that trait, and create the job command and add it a tmp jobs file table=LoadTable(filename=table, header=True, sep='\t') #get dimensions of the table dim=table.Shape created_tmp_files=[] output_files=[] ci_files=[] #create a tmp file to store the job commands (which we will pass to our parallel script to run) jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_asr_') jobs=open(jobs_fp,'w') created_tmp_files.append(jobs_fp) if(verbose): print "Creating temporary input files in: ",tmp_dir #iterate over each column for i in range(1,dim[1]): #create a new table with only a single trait single_col_table=table.getColumns([0,i]) #write the new table to a tmp file single_col_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='in_asr_') single_col_table.writeToFile(single_col_fp,sep='\t') created_tmp_files.append(single_col_fp) #create tmp output files tmp_output_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_') output_files.append(tmp_output_fp) tmp_ci_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='out_asr_ci_') ci_files.append(tmp_ci_fp) #create the job command cmd= "{0} -i {1} -t {2} -m {3} -o {4} -c {5}".format(asr_script_fp, single_col_fp, tree, asr_method, tmp_output_fp, tmp_ci_fp) #add job command to the the jobs file jobs.write(cmd+"\n") jobs.close() created_tmp_files.extend(output_files) created_tmp_files.extend(ci_files) if(verbose): print "Launching parallel jobs." #run the job command job_prefix='asr' submit_jobs(cluster_jobs_fp ,jobs_fp,job_prefix,num_jobs=num_jobs) if(verbose): print "Jobs are now running. Will wait until finished." #wait until all jobs finished (e.g. simple poller) wait_for_output_files(output_files) if(verbose): print "Jobs are done running. Now combining all tmp files." #Combine output files combined_table=combine_asr_tables(output_files) combined_ci_table=combine_asr_tables(ci_files) #create a Table object combined_table=Table(header=combined_table[0],rows=combined_table[1:]) combined_ci_table=Table(header=combined_ci_table[0],rows=combined_ci_table[1:]) #clean up all tmp files for file in created_tmp_files: remove(file) #return the combined table return combined_table,combined_ci_table