def run(options): HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Citation") if (options.option == 'all'): print(colored("\n+ BacterialTyper citation:", 'blue')) only_us() print(colored("\n+ Other softwares employed in the pipeline", 'blue')) print_all() elif (options.option == 'only'): print(colored("\n+ BacterialTyper citation:", 'blue')) only_us() return ()
def run_cluster(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_Mash): ## information for Min Hash Software min_hash_caller.helpMash() exit() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Clustering samples") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default project_mode = True if (options.detached): options.project = False project_mode = False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ## get files if options.reads: if options.noTrim: ## raw reads pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug) else: ## trimm reads pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) ## keep only R1 reads if paired-end if options.pair: pd_samples_retrieved = pd_samples_retrieved.loc[ pd_samples_retrieved['read_pair'] == "R1"] else: ## default pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "assembly", ["fna"], options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) # exit if empty if pd_samples_retrieved.empty: print( "No data has been retrieved from the project folder provided. Exiting now..." ) exit() ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ## for each sample outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "mash", options.debug) ## debug message if (Debug): print(colored("**DEBUG: outdir_dict **", 'yellow')) print(outdir_dict) ## get databases to check retrieve_databases = get_options_db(options) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## remove samples if specified if options.ex_sample: ex_samples = HCGB_main.get_info_file(options.ex_sample) retrieve_databases = retrieve_databases.loc[~retrieve_databases.index. isin(ex_samples)] ## debug message if (Debug): print(colored("**DEBUG: retrieve_database **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print(retrieve_databases) ## check if all samples in user_data or genbank are indexed siglist_all = [] for index, row in retrieve_databases.iterrows(): if not row['path'] == 'NaN': if (Debug): HCGB_aes.print_sepLine("*", 25, False) print(row) if all([ int(options.kmer_size) == int(row['ksize']), int(options.n_sketch) == int(row['num_sketch']) ]): siglist_all.append( min_hash_caller.read_signature(row['path'], options.kmer_size)) continue ## index assembly or reads... (sigfile, siglist) = generate_sketch(row['folder'], row['original'], index, options.kmer_size, options.n_sketch, Debug) retrieve_databases.loc[index]['path'] = sigfile retrieve_databases.loc[index]['ksize'] = options.kmer_size retrieve_databases.loc[index]['num_sketch'] = options.n_sketch siglist_all.append(siglist) ### Cluster project samples print(colored("\n+ Collect project data", 'green')) print("+ Generate mash sketches for each sample analyzed...") pd_samples_retrieved = pd_samples_retrieved.set_index('name') ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieved **", 'yellow')) print(pd_samples_retrieved) ## init dataframe for project data colname = ["source", "name", "path", "original", "ksize", "num_sketch"] pd_samples_sketched = pd.DataFrame(columns=colname) for index, row in pd_samples_retrieved.iterrows(): if index in retrieve_databases.index: print( colored( '\t+ Sketched signature (%s) available within user data...' % index, 'yellow')) continue this_sig = outdir_dict[index] + '/' + index + '.sig' if os.path.exists(this_sig): ## File signature might exist ## read original file2print = outdir_dict[index] + '/.original' if not os.path.exists(file2print): original = ['NaN'] else: original = HCGB_main.readList_fromFile(file2print) if all([ int(options.kmer_size) == int(original[1]), int(options.n_sketch) == int(original[2]) ]): siglist_all.append( min_hash_caller.read_signature(this_sig, options.kmer_size)) pd_samples_sketched.loc[len(pd_samples_sketched)] = ( 'project_data', index, this_sig, row['sample'], options.kmer_size, options.n_sketch) print( colored( '\t+ Sketched signature available (%s) in project folder...' % index, 'green')) continue print( colored('\t+ Sketched signature to be generated: (%s)...' % index, 'yellow')) ## index assembly or reads... (sigfile, siglist) = generate_sketch(outdir_dict[index], row['sample'], index, options.kmer_size, options.n_sketch, Debug) pd_samples_sketched.loc[len(pd_samples_sketched)] = ('project_data', index, sigfile, row['sample'], options.kmer_size, options.n_sketch) siglist_all.append(siglist) print("\n+ Clustering sequences...") pd_samples_sketched = pd_samples_sketched.set_index('name') #### if retrieve_databases.empty: cluster_df = pd_samples_sketched else: tmp = retrieve_databases[[ 'source', 'db', 'path', 'original', 'ksize', 'num_sketch' ]] tmp = tmp.rename(columns={'db': 'name'}) tmp.set_index('name') if (Debug): print(colored("**DEBUG: tmp **", 'yellow')) print(tmp) ## merge both dataframes cluster_df = pd.concat([pd_samples_sketched, tmp], join='inner', sort=True) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_sketched **", 'yellow')) print(pd_samples_sketched) print(colored("**DEBUG: cluster_df **", 'yellow')) print(cluster_df) print(colored("**DEBUG: Signatures **", 'yellow')) print(siglist_all) print(colored("**DEBUG: length siglist_all **", 'yellow')) print(len(siglist_all)) ## Assign Colors colorLabels color_df = cluster_df.filter(["source"], axis=1) color_df["color"] = "r" ## red::genbank ## project data project_data = list(color_df[color_df["source"] == "project_data"].index) color_df.loc[color_df.index.isin(project_data), "color"] = "g" ## green::project_data ## user_data user_data = list(color_df[color_df["source"] == "user_data"].index) color_df.loc[color_df.index.isin(user_data), "color"] = "b" ## blue::user_data colorLabels = color_df['color'].to_dict() if Debug: print(color_df) print(colorLabels) ## parse results if options.project: outdir_report = HCGB_files.create_subfolder("report", outdir) #final_dir = outdir + '/report/cluster' final_dir = functions.create_subfolder("cluster", outdir_report) else: final_dir = outdir ## compare name = 'cluster_' + str(HCGB_time.create_human_timestamp()) tag_cluster_info = final_dir + '/' + name print('+ Saving results in folder: ', final_dir) print('\tFile name: ', name) (DataMatrix, labeltext) = min_hash_caller.compare(siglist_all, tag_cluster_info, Debug) ## get colorLabels ## plot images pdf = True cluster_returned = min_hash_caller.plot(DataMatrix, labeltext, tag_cluster_info, pdf, colorLabels) ## generate newick tree min_hash_caller.get_Newick_tree(cluster_returned, DataMatrix, labeltext, tag_cluster_info) return ()
def run_profile(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option sampleParser.help_format() exit() if (options.help_project): ## information for project help_info.project_help() exit() if (options.help_ARIBA): ## help_format option ariba_caller.help_ARIBA() exit() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ## message header HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Virulence & Resistance profile module") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out options.database = os.path.abspath(options.database) global input_dir input_dir = os.path.abspath(options.input) outdir = "" ## set mode: project/detached global Project if (options.detached): options.project = False outdir = os.path.abspath(options.output_folder) Project = False else: options.project = True outdir = input_dir Project = True ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ## for each sample outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "profile", options.debug) ### print( "+ Generate a sample profile for virulence and resistance candidate genes for each sample retrieved using:" ) print( "(1) Antimicrobial Resistance Inference By Assembly (ARIBA) software") print( "(2) Pre-defined databases by different suppliers or user-defined databases." ) ## get databases to check retrieve_databases = get_options_db(options) ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_total) ######## ARIBA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases, start_time_partial) ###################################### ## update database for later usage ###################################### if not options.fast: ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) HCGB_aes.boxymcboxface("Update Sample Database") ## update db print("+ Update database with samples identified") ## TODO: check if it works dataBase_user = database_user.update_database_user_data( options.database, input_dir, Debug, options) ## debug message if (Debug): print(colored("**DEBUG: results obtained **", 'yellow')) else: print( "+ No update of the database has been requested using option --fast" ) print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting Virulence & Resistance profile module.") return ()
def run_assembly(options): """Main function of the assemble module. It assembles each sample using SPADES_ and checks quality using BUSCO_ software and database. .. seealso:: This function depends on other BacterialTyper and HCGB functions called: - :func:`BacterialTyper.scripts.BUSCO_caller.print_help_BUSCO` - :func:`BacterialTyper.scripts.multiQC_report.multiqc_help` - :func:`BacterialTyper.modules.qc.BUSCO_check` - :func:`HCGB.sampleParser` - :func:`HCGB.functions.aesthetics_functions` - :func:`HCGB.functions.time_functions` - :func:`HCGB.functions.main_functions` - :func:`HCGB.functions.file_functions` .. include:: ../../links.inc """ ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option help_info.help_fastq_format() exit() elif (options.help_BUSCO): ## information for BUSCO BUSCO_caller.print_help_BUSCO() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_multiqc): ## information for Multiqc multiQC_report.multiqc_help() exit() ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ## message header HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Assembly module") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default project_mode = True if (options.detached): options.project = False project_mode = False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "assemble", options.debug) ### call assemble using spades start_time_partial = start_time_total start_time_partial_assembly = start_time_partial ## optimize threads name_list = set(pd_samples_retrieved["name"].tolist()) threads_job = HCGB_main.optimize_threads( options.threads, len(name_list)) ## threads optimization max_workers_int = int(options.threads / threads_job) ## debug message if (Debug): HCGB_aes.debug_message("options.threads: " + str(options.threads), "yellow") HCGB_aes.debug_message("max_workers: " + str(max_workers_int), "yellow") HCGB_aes.debug_message("cpu_here: " + str(threads_job), "yellow") # Group dataframe by sample name sample_frame = pd_samples_retrieved.groupby(["name"]) # We can use a with statement to ensure threads are cleaned up promptly print('+ Running modules SPADES...') with concurrent.futures.ThreadPoolExecutor( max_workers=max_workers_int) as executor: ## send for each sample commandsSent = { executor.submit(check_sample_assembly, name, outdir_dict[name], sorted(cluster["sample"].tolist()), threads_job): name for name, cluster in sample_frame } for cmd2 in concurrent.futures.as_completed(commandsSent): details = commandsSent[cmd2] try: data = cmd2.result() except Exception as exc: print('***ERROR:') print(cmd2) print('%r generated an exception: %s' % (details, exc)) ## functions.timestamp print("\n+ Assembly of all samples finished: ") start_time_partial = HCGB_time.timestamp(start_time_partial_assembly) ## if (assembly_stats): ################### if Debug: HCGB_aes.debug_message("assembly_stats dictionary", "yellow") print(assembly_stats) ## create single file get_assembly_stats_all(assembly_stats, outdir, Debug) ### symbolic links print("+ Retrieve all genomes assembled...") ### BUSCO check assembly if (options.no_BUSCO): print() else: results = qc.BUSCO_check(outdir, outdir, options, start_time_partial, "genome") ## print to file results print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting Assembly module.") return ()
def run_phylo(options): """ Main function acting as an entry point to the module *phylo*. """ ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option sampleParser.help_format() exit() elif (options.help_project): ## information for project help_info.project_help() exit() ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Phylogenetic reconstruction") print ("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir="" ## set mode: project/detached ## Project mode as default project_mode=True if (options.detached): options.project = False project_mode=False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ## get the database options.database = os.path.abspath(options.database) ### parse the reference print ("+ Retrieve the reference...") reference_gbk_file = get_reference_gbk(options) ## generate output folder, if necessary print ("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ################################## ## select samples and map #################################### print ("+ Retrieve samples to map available...") dict_folders = map_samples(options, reference_gbk_file, input_dir, outdir) if Debug: print (colored("**DEBUG: dict_folders **", 'yellow')) print (dict_folders) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ################################## ## Create core alingment ################################## outdir_report = HCGB_files.create_subfolder("report", outdir) phylo_dir = HCGB_files.create_subfolder("phylo", outdir_report) analysis_dir = HCGB_files.create_subfolder(options.name, phylo_dir) snippy_dir = HCGB_files.create_subfolder("snippy", analysis_dir) list_folders = list(dict_folders.values()) options_string = "" variant_calling.snippy_core_call(list_folders, options_string, options.name, snippy_dir, options.output_format, Debug) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## snp distance matrix snp_distance_dir = HCGB_files.create_subfolder("snp_distance", analysis_dir) name_matrix = os.path.join(snp_distance_dir, "snp_matrix_" + options.name) countGaps = False aln_file = os.path.join(snippy_dir, options.name + '.aln') phylo_parser.get_snp_distance(aln_file, options.output_format, countGaps, name_matrix, Debug) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## phylogenetic analysis iqtree_output = HCGB_files.create_subfolder("iqtree", analysis_dir) phylo_parser.ml_tree(snippy_dir, options.name, options.threads, iqtree_output, Debug) ## time stamp start_time_partial = HCGB_files.timestamp(start_time_total) print ("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print ("+ Exiting Annotation module.") return()
def run_ident(options): """ Main function acting as an entry point to the module *ident*. Arguments: .. seealso:: Additional information to PubMLST available datasets. - :doc:`PubMLST datasets<../../../data/PubMLST_datasets>` """ ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option sampleParser.help_format() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_KMA): ## information for KMA Software species_identification_KMA.help_kma_database() exit() elif (options.help_MLSTar): ## information for KMA Software MLSTar.help_MLSTar() exit() ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ### species_identification_KMA -> most similar taxa HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Species identification") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default global Project if (options.detached): options.project = False project_mode = False outdir = os.path.abspath(options.output_folder) Project = False else: options.project = True outdir = input_dir Project = True ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ## for each sample outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "ident", options.debug) ## let's start the process print( "+ Generate an species typification for each sample retrieved using:") print("(1) Kmer alignment (KMA) software.") print("(2) Pre-defined databases by KMA or user-defined databases.") ## get databases to check retrieve_databases = get_options_db(options) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## debug message if (Debug): print(colored("**DEBUG: retrieve_database **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print(retrieve_databases) ######## KMA identification dataFrame_kma = KMA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases, start_time_partial) ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ## debug message if (Debug): print(colored("**DEBUG: retrieve results to summarize **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print("dataframe_kma") print(dataFrame_kma) ## exit if viral search skip = False if (len(options.kma_dbs) == 1): for i in options.kma_dbs: if (i == 'viral'): print() MLST_results = '' options.fast = True skip = True ## what if only plasmids? ## do edirect and MLST if bacteria if (not skip): dataFrame_edirect = pd.DataFrame() ######## EDirect identification #dataFrame_edirect = edirect_ident(dataFrame_kma, outdir_dict, Debug) ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ## debug message if (Debug): print(colored("**DEBUG: retrieve results from NCBI **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print("dataFrame_edirect") print(dataFrame_edirect) ######## MLST identification MLST_results = MLST_ident(options, dataFrame_kma, outdir_dict, dataFrame_edirect, retrieve_databases) ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ## debug message if (Debug): print( colored("**DEBUG: retrieve results to summarize **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print("MLST_results") print(MLST_results) ## generate summary for sample: all databases ## MLST, plasmids, genome, etc HCGB_aes.boxymcboxface("Results Summary") ##################################### ## Summary identification results ## ##################################### ## parse results if options.project: final_dir = os.path.join(outdir, 'report', 'ident') HCGB_files.create_folder(final_dir) else: final_dir = outdir ### excel_folder = HCGB_files.create_subfolder("samples", final_dir) print('+ Print summary results in folder: ', final_dir) print('+ Print sample results in folder: ', excel_folder) # Group dataframe results summary by sample name sample_results_summary = dataFrame_kma.groupby(["Sample"]) ## debug message if (Debug): print(colored("**DEBUG: sample_results_summary **", 'yellow')) print(sample_results_summary) ## results_summary_KMA = pd.DataFrame() MLST_all = pd.DataFrame() for name, grouped in sample_results_summary: ## create a excel and txt for sample name_sample_excel = excel_folder + '/' + name + '_ident.xlsx' name_sample_csv = outdir_dict[ name] + '/ident_summary.csv' ## check in detached mode writer_sample = pd.ExcelWriter( name_sample_excel, engine='xlsxwriter') ## open excel handle ## subset dataframe & print result results_summary_toPrint_sample = grouped[[ 'Sample', '#Template', 'Query_Coverage', 'Template_Coverage', 'Depth', 'Database' ]] results_summary_toPrint_sample.to_excel( writer_sample, sheet_name="KMA") ## write excel handle results_summary_toPrint_sample.to_csv( name_sample_csv) ## write csv for sample ## read MLST if MLST_results: if name in MLST_results: sample_MLST = pd.read_csv(MLST_results[name], header=0, sep=',') sample_MLST['genus'] = dataFrame_edirect.loc[ dataFrame_edirect['sample'] == name, 'genus'].values[0] sample_MLST['species'] = dataFrame_edirect.loc[ dataFrame_edirect['sample'] == name, 'species'].values[0] sample_MLST.to_excel(writer_sample, sheet_name="MLST") ## write excel handle ## Return information to excel MLST_all = pd.concat([MLST_all, sample_MLST]) ## close excel handle writer_sample.save() ## name_excel = final_dir + '/identification_summary.xlsx' print('+ Summary information in excel file: ', name_excel) writer = pd.ExcelWriter(name_excel, engine='xlsxwriter') ## open excel handle ## KMA dataframe: print result for sources results_summary_KMA = dataFrame_kma[[ 'Sample', '#Template', 'Query_Coverage', 'Template_Coverage', 'Depth', 'Database' ]] ## Sum plasmid and chromosome statistics ## ## sum coverage total_coverage = results_summary_KMA.groupby( 'Sample')['Query_Coverage'].sum().reset_index() ## debug message if (Debug): print("*** Sum: Query_coverage ***") print(total_coverage) ## TODO: FIX SUMMARY REPORT results_summary_KMA = results_summary_KMA.set_index('Sample') results_summary_KMA = results_summary_KMA.sort_values( by=['Sample', 'Database', 'Query_Coverage'], ascending=[True, True, True]) results_summary_KMA.to_excel(writer, sheet_name='KMA') ## write excel handle ## write MLST if (MLST_results): MLST_all.to_excel(writer, sheet_name='MLST') ## write excel and close writer.save() ## close excel handle print("\n+ Check summary of results in file generated") ### timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ###################################### ## update database for later usage ###################################### if not options.fast: HCGB_aes.boxymcboxface("Update Sample Database") ## update db print("+ Update database with samples identified") ## debug message if (Debug): print(colored("**DEBUG: dataFrame_edirect **", 'yellow')) pd.set_option('display.max_colwidth', None) pd.set_option('display.max_columns', None) print(dataFrame_edirect) ## dataFrame_edirect file_toprint = final_dir + '/edirect_info2download.csv' dataFrame_edirect.to_csv(file_toprint) ## update database with samples identified data2download = dataFrame_edirect.filter( ['genus', 'species', 'strain', 'genome']) data2download = data2download.rename(columns={ 'genome': 'NCBI_assembly_ID', 'strain': 'name' }) NCBI_folder = os.path.abspath(options.database) + '/NCBI' database_generator.NCBI_DB(data2download, NCBI_folder, Debug) else: print( "+ No update of the database has been requested using option --fast" ) print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting identification module.") return ()
def run_BacterialTyper(options): ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option help_info.help_fastq_format() exit() elif (options.help_BUSCO): ## information for BUSCO BUSCO_caller.print_help_BUSCO() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_multiqc): ## information for Multiqc multiQC_report.multiqc_help() elif (options.help_Prokka): ## information for Prokka annotation.print_list_prokka() exit() elif (options.help_Mash): ## information for Min Hash Software min_hash_caller.helpMash() exit() elif (options.help_ARIBA): ## information for ARIBA ariba_caller.help_ARIBA() exit() elif (options.help_trimm_adapters): ## help on trimm adapters trimmomatic_call.print_help_adapters() exit() elif (options.help_KMA): ## information for KMA Software species_identification_KMA.help_kma_database() exit() elif (options.help_MLSTar): ## information for KMA Software MLSTar.help_MLSTar() exit() elif (options.help_PhiSpy): ## information for PhiSpy software bacteriophage.help_PhiSpy() exit() elif (options.help_MGE_analysis): ## information for MGE module analysis MGE.help_MGE_analysis() exit() elif (options.help_input_MGE): ## information for PhiSpy MGE.help_input_MGE() exit() ### HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("BacterialTyper analysis") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## set mode: project/detached if (options.project): outdir = input_dir elif (options.detached): outdir = os.path.abspath(options.output_folder)
def run_database(options): ## init time start_time_total = time.time() start_time_partial = start_time_total ## debugging messages global Debug if (options.debug): Debug = True print("[Debug mode: ON]") else: Debug = False ## message header HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Database") print("--------- Starting Process ---------") HCGB_time.print_time() kma_bin = set_config.get_exe("kma") ###################################################### ## print further information if requested if (options.help_ARIBA): print("ARIBA databases information:") ariba_caller.help_ARIBA() exit() elif (options.help_BUSCO): BUSCO_caller.print_help_BUSCO() exit() elif (options.help_KMA): species_identification_KMA.help_kma_database() exit() ###################################################### ## create folder ## absolute options.path = os.path.abspath(options.path) HCGB_files.create_folder(options.path) ######### if Debug: print(colored("DEBUG: absolute path folder: " + options.path, 'yellow')) ########## ## NCBI ## ########## ## if any NCBI options provided if any([options.ID_file, options.descendant]): ## create folders NCBI_folder = HCGB_files.create_subfolder('NCBI', options.path) if (options.ID_file): ## get path and check if it is file abs_path_file = os.path.abspath(options.ID_file) if os.path.isfile(abs_path_file): print() HCGB_aes.print_sepLine("*", 50, False) print("--------- Check NCBI ids provided ---------\n") HCGB_aes.print_sepLine("*", 70, False) ## get file information print("\t+ Obtaining information from file: %s" % abs_path_file) strains2get = HCGB_main.get_data(abs_path_file, ',', '') dataBase_NCBI = database_generator.NCBI_DB( strains2get, NCBI_folder, Debug) ######### if Debug: print(colored("DEBUG: NCBI data provided: ", 'yellow')) print(options.ID_file) ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ## strains downloaded would be included to a kma index ## Get all entries belonging to this taxon provided if (options.descendant): ######### if Debug: print(colored("DEBUG: NCBI descendant option: ON ", 'yellow')) print() HCGB_aes.print_sepLine("*", 70, False) print( "--------- Check descendant NCBI taxonomy ids provided ---------\n" ) HCGB_aes.print_sepLine("*", 70, False) ## [TODO] dataBase_NCBI = database_generator.NCBI_descendant( options.descendant, NCBI_folder, Debug) ############################################################## ## update KMA database with NCBI information retrieved ############################################################## print('\n\n+ Update database for later identification analysis...') list_of_files = dataBase_NCBI['genome'].tolist() kma_db = HCGB_files.create_subfolder('KMA_db', options.path) genbank_kma_db = HCGB_files.create_subfolder('genbank', kma_db) print('+ Database to update: ', genbank_kma_db) species_identification_KMA.generate_db(list_of_files, 'genbank_KMA', genbank_kma_db, 'new', 'batch', Debug, kma_bin) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ############### ## user_data ## ############### if options.project_folder: ## dataBase_user = pd.DataFrame() ## get absolute path abs_project_folder = os.path.abspath(options.project_folder) if os.path.exists(abs_project_folder): ######### if Debug: print( colored("DEBUG: User provides folder containing project", 'yellow')) print() HCGB_aes.print_sepLine("*", 70, False) print("--------- Check user provided project folder ---------") HCGB_aes.print_sepLine("*", 70, False) dataBase_user = database_user.update_database_user_data( options.path, abs_project_folder, Debug, options) else: print( colored( "ERROR: Folder provided does not exists: %s" % options.project_folder, 'red')) exit() ############################################################## ## update KMA database with user_data information retrieved ############################################################## print('\n\n+ Update database for later identification analysis...') list_of_files = dataBase_user['genome'].tolist() kma_db = HCGB_files.create_subfolder('KMA_db', options.path) user_kma_db = HCGB_files.create_subfolder('user_data', kma_db) print('+ Database to update: ', user_kma_db) species_identification_KMA.generate_db(list_of_files, 'userData_KMA', user_kma_db, 'new', 'batch', Debug, kma_bin) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ########## ## ARIBA ########## print() HCGB_aes.print_sepLine("*", 50, False) print("--------- Check ARIBA parameters provided --------") HCGB_aes.print_sepLine("*", 50, False) if (options.no_ARIBA): print("+ No ARIBA databases would be downloaded...") ######### if Debug: print(colored("DEBUG: No option ARIBA", 'yellow')) else: #functions.print_sepLine("*",50, False) ### ariba list databases ariba_dbs_list = ['CARD', 'VFDB'] if (options.no_def_ARIBA): ariba_dbs_list = options.ariba_dbs else: if (options.ariba_dbs): ariba_dbs_list = ariba_dbs_list + options.ariba_dbs ariba_dbs_list = set(ariba_dbs_list) ######### if Debug: print(colored("DEBUG: Option ARIBA", 'yellow')) print(options.ariba_dbs) ariba_caller.download_ariba_databases(ariba_dbs_list, options.path, Debug, options.threads) ### ariba list databases if (options.ariba_users_fasta): print( "+ Generate ARIBA database for databases provided: prepare fasta and metadata information" ) ######### if Debug: print(colored("DEBUG: Option user ARIBA db", 'yellow')) print(ariba_users_fasta) print(ariba_users_meta) ## [TODO]: ## ariba prepareref fasta and metadata ### timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ######### ## kma ## ######### print() HCGB_aes.print_sepLine("*", 50, False) print("--------- Check KMA parameters provided ----------") kma_database = options.path + '/KMA_db' HCGB_files.create_folder(kma_database) ## types: bacteria, archaea, protozoa, fungi, plasmids, typestrains ## downloads all "bacterial" genomes from KMA website ## kma: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/ print( "+ Retrieving information from: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder website" ) ## KMA databases to use ## only user dbs if (options.no_def_kma): if (options.kma_dbs): print("+ Only user databases selected will be indexed...") else: print("+ No databases selected.") print(colored("ERROR: Please select a kma database.", 'red')) exit() ## default dbs + user else: kma_dbs = ["bacteria", "plasmids"] ## default dbs + user if (options.kma_dbs): options.kma_dbs = options.kma_dbs + kma_dbs options.kma_dbs = set(options.kma_dbs) else: options.kma_dbs = kma_dbs ######### if Debug: print(colored("DEBUG: options.kma_dbs", 'yellow')) print(options.kma_dbs) ## Get databases for db in options.kma_dbs: print(colored("\n+ " + db, 'yellow')) db_folder = HCGB_files.create_subfolder(db, kma_database) species_identification_KMA.download_kma_database(db_folder, db, Debug) ### timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) ########### ## BUSCO ## ########### if (options.BUSCO_dbs): print() HCGB_aes.print_sepLine("*", 50, False) print("--------- Check BUSCO datasets provided ---------") BUSCO_folder = HCGB_files.create_subfolder("BUSCO", options.path) ######### if Debug: print(colored("DEBUG: options.BUSCO_dbs", 'yellow')) print(options.BUSCO_dbs) print("+ BUSCO datasets would be downloaded when executed...") #BUSCO_caller.BUSCO_retrieve_sets(options.BUSCO_dbs, BUSCO_folder) ### timestamp start_time_partial = HCGB_time.timestamp(start_time_partial) print("\n*************** Finish *******************\n") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting Database module.\n") return ()
def run_annotation(options): ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option sampleParser.help_format() exit() elif (options.help_BUSCO): ## information for BUSCO BUSCO_caller.print_help_BUSCO() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_multiqc): ## information for Multiqc multiQC_report.multiqc_help() elif (options.help_Prokka): ## information for Prokka annotation.print_list_prokka() exit() ## set default options.batch = False ### HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Assembly annotation") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default project_mode = True if (options.detached): options.project = False project_mode = False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ### symbolic links print("+ Retrieve all genomes assembled...") ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "assembly", ["fna"], options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ## for samples outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "annot", options.debug) ## annotate print("+ Annotate assemblies using prokka:") print("\t-Option: kingdom = ", options.kingdom, "; Annotation mode") if options.genera == 'Other': print( "\t-Option: genera = Off; No genus-specific BLAST databases option provided" ) else: print("\t-Option: genera = ", options.genera, "; Genus-specific BLAST databases option provided") print("\t-Option: addgenes; Add 'gene' features for each 'CDS' feature") print("\t-Option: addmrna; Add 'mRNA' features for each 'CDS' feature") print("\t-Option: cdsrnaolap; Allow [tr]RNA to overlap CDS") ## optimize threads name_list = set(pd_samples_retrieved["name"].tolist()) threads_job = HCGB_main.optimize_threads( options.threads, len(name_list)) ## threads optimization max_workers_int = int(options.threads / threads_job) ## debug message if (Debug): print( colored("**DEBUG: options.threads " + str(options.threads) + " **", 'yellow')) print( colored("**DEBUG: max_workers " + str(max_workers_int) + " **", 'yellow')) print( colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow')) ## send for each sample with concurrent.futures.ThreadPoolExecutor( max_workers=max_workers_int) as executor: commandsSent = { executor.submit(annot_caller, row['sample'], outdir_dict[row['name']], options, row['name'], threads_job): index for index, row in pd_samples_retrieved.iterrows() } for cmd2 in concurrent.futures.as_completed(commandsSent): details = commandsSent[cmd2] try: data = cmd2.result() except Exception as exc: print('***ERROR:') print(cmd2) print('%r generated an exception: %s' % (details, exc)) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## get folders givenList = [v for v in outdir_dict.values()] protein_files = [] print( "+ Detail information for each sample could be identified in separate folders:" ) for folder in givenList: print('\t + ', folder) protein_files.extend( HCGB_main.retrieve_matching_files(folder, '.faa', Debug)) ### report generation if (options.skip_report): print("+ No annotation report generation...") else: ### report generation HCGB_aes.boxymcboxface("Annotation report") outdir_report = HCGB_files.create_subfolder("report", outdir) PROKKA_report = HCGB_files.create_subfolder("annotation", outdir_report) print( '\n+ A summary HTML report of each sample is generated in folder: %s' % PROKKA_report) ## check if previously report generated filename_stamp = PROKKA_report + '/.success' done = 0 if os.path.isdir(PROKKA_report): if os.path.isfile(filename_stamp): stamp = HCGB_time.read_time_stamp(filename_stamp) print( colored( "\tA previous report generated results on: %s" % stamp, 'yellow')) done = 1 ## generate report if done == 0: ## get subdirs generated and call multiQC report module multiQC_report.multiQC_module_call(givenList, "Prokka", PROKKA_report, "-dd 2") print( '\n+ A summary HTML report of each sample is generated in folder: %s' % PROKKA_report) ## success stamps filename_stamp = PROKKA_report + '/.success' stamp = HCGB_time.print_time_stamp(filename_stamp) ## time stamp start_time_partial_BUSCO = HCGB_time.timestamp(start_time_total) ## Check each annotation using BUSCO results = qc.BUSCO_check(input_dir, outdir, options, start_time_partial_BUSCO, "proteins") ## print to file: results print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting Annotation module.") return ()
def run_report(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_spaTyper): ## help_format option get_spa_typing.help_spaTyper() exit() elif (options.help_project): ## information for project help_info.project_help() exit() ## set default options.batch = False ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ## message header HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Report generation module") print("--------- Starting Process ---------") HCGB_time.print_time() ## call assemble using spades start_time_partial = start_time_total ## absolute path for in & out options.database = os.path.abspath(options.database) global input_dir input_dir = os.path.abspath(options.input) outdir = "" ## set mode: project/detached global Project if (options.detached): options.project = False outdir = os.path.abspath(options.output_folder) Project = False else: options.project = True outdir = input_dir Project = True ## print("\n+ Get project information:") ## get files: trimm, assembly, annotation pd_samples_retrieved = database_user.get_userData_files(options, input_dir) pd_samples_retrieved['new_name'] = pd_samples_retrieved['name'] ## get info: profile, ident, cluster, MGE pd_samples_info = database_user.get_userData_info(options, input_dir) ## get databases to list #retrieve_databases = get_options_db(options) ## create output files outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "report", options.debug) ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) print(colored("**DEBUG: pd_samples_info **", 'yellow')) print(pd_samples_info) ## generate output folder, if necessary print( "\n\n\n+ Generate a report summarizing analysis and sample information" ) if not options.project: HCGB_files.create_folder(outdir) outdir_report = outdir else: ### report generation outdir_report = HCGB_files.create_subfolder("report", outdir) ## create report with all data summary_report = HCGB_files.create_subfolder("summary_report", outdir_report) print("Folder: ", summary_report) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_partial) ######################################## ## create species specific report if any ######################################## if (options.species_report): ## Saureus if options.species_report == "Saureus": Saureus_specific(pd_samples_retrieved, pd_samples_info, options, summary_report, outdir_dict) ## else ## to add accordingly ## time stamp start_time_partial = HCGB_time.timestamp(start_time_partial) ########################################################### ## create gene fasta sequences retrieval if desired ########################################################### if options.genes_ids_fasta: ## given a list of genes ids, retrieve sequence for all samples from profile if os.path.isfile(os.path.abspath(options.genes_ids_fasta)): in_file = os.path.abspath(options.genes_ids_fasta) gene_names = [line.rstrip('\n') for line in open(in_file)] print( '+ Retrieve selected genes sequences from the profile analysis for each sample.' ) print('+ Searching gene:') ## get profiles available results_geneIDs = pd.DataFrame(columns=('sample', 'gene', 'id', 'sequence')) sample_frame = pd_samples_info.groupby(["name"]) for g in gene_names: print("\t+", g) for name, cluster_df in sample_frame: my_list_profiles = cluster_df.loc[ cluster_df['tag'] == 'profile']['ext'].to_list() if options.debug: print("name: ", name) print("my_list_profiles:") print(my_list_profiles) for p in my_list_profiles: main_profile_folder = cluster_df.loc[ cluster_df['ext'] == p]['dirname'].to_list()[0] p = p.lower() if p == 'vfdb': p = p + '_full' profile_folder = os.path.join(main_profile_folder, p) (seq_id, seq_sequence ) = retrieve_genes.retrieve_genes_ids_sequences( profile_folder, g, Debug) if (seq_id): ## save results results_geneIDs.loc[len(results_geneIDs)] = ( name, g, seq_id, seq_sequence) ## save for each gene in a separate fasta file list_of_genes = set(results_geneIDs['gene'].to_list()) ## debug if Debug: print("** DEBUG **") print(results_geneIDs) print(list_of_genes) ## Save results genes_folder = HCGB_files.create_subfolder('genes', summary_report) for gene_retrieved in list_of_genes: this_frame = results_geneIDs[results_geneIDs['gene'] == gene_retrieved] gene_retrieved_file = os.path.join(genes_folder, gene_retrieved) gene_retrieved_fasta = gene_retrieved_file + ".fasta" gene_retrieved_info = gene_retrieved_file + "_info.txt" fasta_hd = open(gene_retrieved_fasta, 'w') info_hd = open(gene_retrieved_info, 'w') for item, row in this_frame.iterrows(): string2write = ">" + row['sample'] + '_' + row[ 'gene'] + '\n' + row['sequence'] + '\n' string2write_info = row['sample'] + '\t' + row[ 'gene'] + '\t' + row['id'] + '\n' fasta_hd.write(string2write) info_hd.write(string2write_info) fasta_hd.close() info_hd.close() ## time stamp start_time_partial = HCGB_time.timestamp(start_time_partial) ######################################## ## create gene promoter fasta sequences retrieval if desired ######################################## if options.promoter_bp: ## retrieve as many bp as necessary from genes_ids_fasta print("** THIS OPTION IS NOT IMPLEMENTED YET... **") #get_promoter.get_promoter(file, geneOfInterest, basePairs, sampleName, option, debug=False): ######################################## ## create gene specific report if any ######################################## if options.genes_ids_profile: if options.species_report == "Saureus": if Debug: print("** options.genes_ids_profile **") print("Analysis already done for Saureus") else: in_file = os.path.abspath(options.genes_ids_profile) gene_names = [line.rstrip('\n') for line in open(in_file)] results_Profiles = retrieve_genes.get_genes_profile( pd_samples_info, gene_names, options.debug, "name") if options.debug: print("results_Profiles") print(results_Profiles) ## open excel writer name_excel = summary_report + '/gene_ids_profile.xlsx' writer = pd.ExcelWriter(name_excel, engine='xlsxwriter') results_Profiles.to_excel(writer, sheet_name="gene_ids") ## close writer.save() ## time stamp start_time_partial = HCGB_time.timestamp(start_time_partial) ############################################### ## Search for any additional fasta sequence ############################################### if options.genes_fasta: ## given a list of fasta sequences search using blast against proteins annotated or genome print("** THIS OPTION IS NOT IMPLEMENTED YET... **") print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("+ Exiting Report generation module.") return ()
def run(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option help_info.help_fastq_format() exit() elif (options.help_trimm_adapters): ## help on trimm adapters trimmomatic_call.print_help_adapters() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_multiqc): ## information for Multiqc multiQC_report.multiqc_help() exit() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Trimming samples") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default if (options.detached): options.project = False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug) ## debug message if (Debug): HCGB_aes.debug_message("pd_samples_retrieved", 'yellow') HCGB_main.print_all_pandaDF(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: HCGB_files.create_folder(outdir) ## for samples outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "trimm", options.debug) ## optimize threads name_list = set(pd_samples_retrieved["name"].tolist()) threads_job = HCGB_main.optimize_threads( options.threads, len(name_list)) ## threads optimization max_workers_int = int(options.threads / threads_job) ## debug message if (Debug): print( colored("**DEBUG: options.threads " + str(options.threads) + " **", 'yellow')) print( colored("**DEBUG: max_workers " + str(max_workers_int) + " **", 'yellow')) print( colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow')) print("+ Trimming adapters for each sample retrieved...") # Group dataframe by sample name sample_frame = pd_samples_retrieved.groupby(["name"]) # Trimming adapters if (options.adapters): # Adapter file provided options.adapters = os.path.abspath(options.adapters) print("\t- Adapters file provided...") else: # Get default adpaters file print("\t- Default Trimmomatic adapters (v0.39) will be used...") options.adapters = data_files.data_list( "available_Trimmomatic_adapters") ## send for each sample with concurrent.futures.ThreadPoolExecutor( max_workers=max_workers_int) as executor: commandsSent = { executor.submit(trimmo_caller, sorted(cluster["sample"].tolist()), outdir_dict[name], name, threads_job, Debug, options.adapters): name for name, cluster in sample_frame } for cmd2 in concurrent.futures.as_completed(commandsSent): details = commandsSent[cmd2] try: data = cmd2.result() except Exception as exc: print('***ERROR:') print(cmd2) print('%r generated an exception: %s' % (details, exc)) print("\n\n+ Trimming samples has finished...") ## functions.timestamp start_time_partial = HCGB_time.timestamp(start_time_total) ## get files generated and generate symbolic link if not options.project: dir_symlinks = HCGB_files.create_subfolder('link_files', outdir) files2symbolic = [] folders = os.listdir(outdir) ## debug message if (Debug): print( colored( "**DEBUG: generate symbolic links for each file in " + dir_symlinks + "**", 'yellow')) for fold in folders: if fold.endswith(".log"): continue else: this_folder = outdir + '/' + fold subfiles = os.listdir(this_folder) for files in subfiles: files_search = re.search( r".*trim_R\d{1}.*", files) ## only paired-end. Todo: single end if files_search: files2symbolic.append(this_folder + '/' + files) HCGB_files.get_symbolic_link(files2symbolic, dir_symlinks) if (options.skip_report): print("+ No report generation...") else: print("\n+ Generating a report using MultiQC module.") outdir_report = HCGB_files.create_subfolder("report", outdir) ## call multiQC report module givenList = [v for v in outdir_dict.values()] my_outdir_list = set(givenList) ## debug message if (Debug): HCGB_aes.debug_message("my_outdir_list for multiqc report", "yellow") print(my_outdir_list) print("\n") trimm_report = HCGB_files.create_subfolder("trimm", outdir_report) multiQC_report.multiQC_module_call(my_outdir_list, "Trimmomatic", trimm_report, "") print( '\n+ A summary HTML report of each sample is generated in folder: %s' % trimm_report) ## create fastqc for trimmed reads pd_samples_retrieved_trimmed = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) qc.fastqc(pd_samples_retrieved_trimmed, outdir, options, start_time_partial, "trimmed", Debug) print("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print("\n+ Exiting trimm module.") return ()
def run_biotype(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option help_XICRA.help_fastq_format() elif (options.help_project): ## information for project help_XICRA.project_help() exit() elif (options.help_RNAbiotype): ## information for join reads RNAbiotype.help_info() exit() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True aesthetics_functions.pipeline_header('XICRA') aesthetics_functions.boxymcboxface("RNA biotype analysis") print("--------- Starting Process ---------") time_functions.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## set mode: project/detached if (options.detached): outdir = os.path.abspath(options.output_folder) options.project = False else: options.project = True outdir = input_dir ## get files print('+ Getting files from input folder... ') ## get files if options.noTrim: print('+ Mode: fastq.\n+ Extension: ') print("[ fastq, fq, fastq.gz, fq.gz ]\n") pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug) else: print('+ Mode: trim.\n+ Extension: ') print("[ _trim_ ]\n") pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) ## Discard if joined reads: use trimmed single-end or paired-end pd_samples_retrieved = pd_samples_retrieved[ pd_samples_retrieved['ext'] != '_joined'] ## debug message if (Debug): print(colored("**DEBUG: pd_samples_retrieve **", 'yellow')) print(pd_samples_retrieved) ## generate output folder, if necessary print("\n+ Create output folder(s):") if not options.project: files_functions.create_folder(outdir) ## for samples mapping_outdir_dict = files_functions.outdir_project( outdir, options.project, pd_samples_retrieved, "map", options.debug) ## debug message if (Debug): print(colored("**DEBUG: mapping_outdir_dict **", 'yellow')) print(mapping_outdir_dict) # time stamp start_time_partial = time_functions.timestamp(start_time_total) ## optimize threads name_list = set(pd_samples_retrieved["new_name"].tolist()) threads_job = main_functions.optimize_threads( options.threads, len(name_list)) ## threads optimization max_workers_int = int(options.threads / threads_job) ## debug message if (Debug): print( colored("**DEBUG: options.threads " + str(options.threads) + " **", 'yellow')) print( colored("**DEBUG: max_workers " + str(max_workers_int) + " **", 'yellow')) print( colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow')) ############################################## ## map Reads ############################################## start_time_partial = mapReads_module(options, pd_samples_retrieved, mapping_outdir_dict, options.debug, max_workers_int, threads_job, start_time_partial, outdir) ## debug message if (Debug): print(colored("**DEBUG: mapping_results **", 'yellow')) print(mapping_results) # time stamp start_time_partial = time_functions.timestamp(start_time_partial) ## for samples biotype_outdir_dict = files_functions.outdir_project( outdir, options.project, pd_samples_retrieved, "biotype", options.debug) ## debug message if (Debug): print(colored("**DEBUG: biotype_outdir_dict **", 'yellow')) print(biotype_outdir_dict) ## get RNAbiotype information RNAbiotype.RNAbiotype_module_call(mapping_results, biotype_outdir_dict, options.annotation, options.debug, max_workers_int, threads_job) # time stamp start_time_partial = time_functions.timestamp(start_time_partial) if (options.skip_report): print("+ No report generation...") else: print( "\n+ Generating a report using MultiQC module for featureCount analysis." ) outdir_report = files_functions.create_subfolder("report", outdir) ## get subdirs generated and call multiQC report module givenList = [] print( "+ Detail information for each sample could be identified in separate folders:" ) ## call multiQC report module givenList = [v for v in biotype_outdir_dict.values()] my_outdir_list = set(givenList) ## debug message if (Debug): print( colored("\n**DEBUG: my_outdir_list for multiqc report **", 'yellow')) print(my_outdir_list) print("\n") featureCount_report = files_functions.create_subfolder( "featureCount", outdir_report) multiQC_report.multiQC_module_call(my_outdir_list, "featureCount", featureCount_report, "-dd 2") print( '\n+ A summary HTML report of each sample is generated in folder: %s' % featureCount_report) ### Summarizing RNA biotype information biotype_report = files_functions.create_subfolder( "biotype", outdir_report) single_files_biotype = files_functions.create_subfolder( "samples", biotype_report) ## results dict_files = {} for samples in biotype_outdir_dict: featurecount_file = os.path.join(biotype_outdir_dict[samples], 'featureCount.out.tsv') if files_functions.is_non_zero_file(featurecount_file): dict_files[samples] = featurecount_file ## copy pdf pdf_plot = main_functions.retrieve_matching_files( biotype_outdir_dict[samples], '.pdf', options.debug) if files_functions.is_non_zero_file(pdf_plot[0]): shutil.copy(pdf_plot[0], single_files_biotype) ## collapse all information all_data = RNAbiotype.generate_matrix(dict_files) ## print into excel/csv print('+ Table contains: ', len(all_data), ' entries\n') ## debugging messages if Debug: print("** DEBUG: all_data") print(all_data) ## set abs_csv_outfile to be in report folder ## copy or link files for each sample analyzed abs_csv_outfile = os.path.join(biotype_report, "summary.csv") all_data.to_csv(abs_csv_outfile) ## create plot: call R [TODO: implement in python] outfile_pdf = os.path.join(biotype_report, "RNAbiotypes_summary.pdf") ## R scripts biotype_R_script = tools.R_scripts('plot_RNAbiotype_sum', options.debug) rscript = set_config.get_exe("Rscript", options.debug) cmd_R_plot = "%s %s -f %s -o %s" % (rscript, biotype_R_script, abs_csv_outfile, outfile_pdf) ## print("+ Create summary plot for all samples") callCode = system_call_functions.system_call(cmd_R_plot) print("\n*************** Finish *******************") start_time_partial = time_functions.timestamp(start_time_total) print("\n+ Exiting join module.") return ()
def run(options): """ This is the main function of the module ``config``. It basically checks if the different requirements (python` and third-party software) are fulfilled. If any requirement is not available this modules tries to install them or reports to the user to manually install them. :param option: State whether to check or install missing modules, packages and third party software. Provide: check/install :param install_path: Absolute path to install modules or packages missing. Default: ``BacterialTyper`` environment path. :param IslandPath: True/False for checking additional perl and software required by this option analysis. :param debug: True/false for debugging messages. :type option: string :type IslandPath: boolean :type install_path: string :type debug: boolean .. seealso:: This function depends on several ``BacterialTyper`` functions: - :func:`BacterialTyper.config.set_config.check_python_packages` - :func:`BacterialTyper.config.set_config.check_perl_packages` - :func:`BacterialTyper.config.extern_progs.return_min_version_soft` - :func:`BacterialTyper.config.extern_progs.print_dependencies` """ ## init time start_time_total = time.time() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Pipeline Configuration") print("--------- Starting Process ---------") HCGB_time.print_time() if (options.install_path): if os.path.isdir(options.install_path): if (Debug): print( "Installation path provided for missing modules, packages, dependencies..." ) print("Path: " + options.install_path) else: print(colored("\n*** ERROR ****", 'red')) print(colored("Path provided is not a folder", 'red')) print(options.install_path) exit() else: ## get python environment path env_bin_directory = os.path.dirname(os.environ['_']) ##os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'templates')) options.install_path = os.path.abspath( os.path.join(env_bin_directory, '../software')) if (Debug): print("Retrieve environment path as installation path:") print("Path: " + options.install_path) HCGB_files.create_folder(options.install_path) ####################### ## install or only check ####################### option_install = False if (options.option == 'install'): print("\n+ Check dependencies") print( "+ Try to install all missing dependencies, modules or third party software..." ) option_install = True ## check if access and permission if os.path.isdir(options.install_path): if (set_config.access_check(options.install_path, mode=os.F_OK)): print( "Installation path is accessible and has permission for installation if necessary" ) else: print(colored("\n*** ERROR ****", 'red')) print( colored( "No access/permission for this path: %s" % options.install_path, 'red')) print( colored( "Please provide a valid path with access/permission to install any missing dependencies.", 'red')) exit() else: print(colored("\n*** ERROR ****", 'red')) print(colored("Path provided is not a folder", 'red')) print(options.install_path) exit() elif (options.option == 'only_check'): print( "\nCheck dependencies, modules or third party software and print report..." ) ####################### ## python version ####################### HCGB_aes.print_sepLine("+", 20, False) print('Python:') HCGB_aes.print_sepLine("+", 20, False) this_python_version = str(sys.version) python_min_version = extern_progs.return_min_version_soft('python') if LooseVersion(this_python_version) >= LooseVersion(python_min_version): print( colored( "Minimum version (%s) satisfied: %s" % (python_min_version, this_python_version), 'green')) else: print( colored( "Minimum version (%s) not satisfied: %s" % (python_min_version, this_python_version), 'red')) exit() ####################### ## perl_version ####################### print('\n') HCGB_aes.print_sepLine("+", 50, False) print('Perl:') HCGB_aes.print_sepLine("+", 50, False) perl_min_version = extern_progs.return_min_version_soft('perl') this_perl_path = set_config.get_exe("perl", Debug) this_perl_version = set_config.get_version("perl", this_perl_path, Debug) if LooseVersion(this_perl_version) >= LooseVersion(perl_min_version): print( colored( "Minimum version (%s) satisfied: %s" % (perl_min_version, this_perl_version), 'green')) else: print( colored( "Minimum version (%s) not satisfied: %s" % (perl_min_version, this_perl_version), 'red')) exit() ####################### ## third-party software ####################### print('\n') HCGB_aes.print_sepLine("+", 20, False) print('External dependencies:') HCGB_aes.print_sepLine("+", 20, False) set_config.check_dependencies(option_install, options.install_path, Debug) print('\n') ####################### ## python packages ####################### print('\n') HCGB_aes.print_sepLine("+", 20, False) print('Python packages:') HCGB_aes.print_sepLine("+", 20, False) set_config.check_python_packages(Debug, option_install, options.install_path) HCGB_aes.print_sepLine("+", 20, False) print('\n') ####################### ## perl packages ####################### print('\n') HCGB_aes.print_sepLine("+", 20, False) print('Perl packages:') HCGB_aes.print_sepLine("+", 20, False) set_config.check_perl_packages("perl_dependencies", Debug, option_install, options.install_path) HCGB_aes.print_sepLine("+", 20, False) print('\n') ####################### ## IslandPath dependencies ####################### if (options.IslandPath): print('\n') HCGB_aes.print_sepLine("+", 20, False) print('IslandPath packages and software required:') HCGB_aes.print_sepLine("+", 20, False) set_config.check_IslandPath(Debug, option_install, options.install_path) HCGB_aes.print_sepLine("+", 20, False) print('\n') ####################### ## R packages ####################### print('\n') HCGB_aes.print_sepLine("+", 20, False) print('R packages:') HCGB_aes.print_sepLine("+", 20, False) set_config.check_R_packages(option_install, options.install_path, Debug) HCGB_aes.print_sepLine("+", 20, False) print('\n')
def run_QC(options): ## init time start_time_total = time.time() ################################## ### show help messages if desired ################################## if (options.help_format): ## help_format option help_info.help_fastq_format() exit() elif (options.help_BUSCO): ## information for BUSCO BUSCO_caller.print_help_BUSCO() exit() elif (options.help_project): ## information for project help_info.project_help() exit() elif (options.help_multiqc): ## information for Multiqc multiQC_report.multiqc_help() exit() ## debugging messages global Debug if (options.debug): Debug = True else: Debug = False ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ## set main header HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Quality check") print("--------- Starting Process ---------") HCGB_time.print_time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = "" ## Project mode as default if (options.detached): options.project = False outdir = os.path.abspath(options.output_folder) else: options.project = True outdir = input_dir ### option if (options.raw_reads): ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug) fastqc(pd_samples_retrieved, outdir, options, start_time_total, "raw", Debug) elif (options.trim_reads): ## get files pd_samples_retrieved = sampleParser.files.get_files( options, input_dir, "trim", ['_trim'], options.debug) fastqc(pd_samples_retrieved, outdir, options, start_time_total, "trimmed", Debug) elif (options.assembly): BUSCO_check(input_dir, outdir, options, start_time_total, "genome") elif (options.annotation): BUSCO_check(input_dir, outdir, options, start_time_total, "proteins") return ()
def run_prep(options): """ Main function of the prep module. This module prepares fastq files for later usage. It initially checks the length of the name and advises the user to rename samples if exceeded. Along ``BacterialTyper`` there are a few string length limitations by different software that need to be sort out from the beginning of the process. This module allows to user to copy files into the project folder initiate or only link using a symbolic link to avoid duplicated raw data. See additional details of this module in user_guide :ref:`prep module entry<prep-description>`. .. seealso:: This function depends on other HCGB functions called: - :func:`HCGB.sampleParser` - :func:`HCGB.functions.aesthetics_functions` - :func:`HCGB.functions.time_functions` - :func:`HCGB.functions.main_functions` - :func:`HCGB.functions.file_functions` """ ## help_format option if (options.help_format): help_info.help_fastq_format() exit() HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version) HCGB_aes.boxymcboxface("Preparing samples") print ("--------- Starting Process ---------") HCGB_time.print_time() ## init time start_time_total = time.time() ## absolute path for in & out input_dir = os.path.abspath(options.input) outdir = os.path.abspath(options.output_folder) ### set as default paired_end mode if (options.single_end): options.pair = False else: options.pair = True ## Project mode as default project_mode=True if (options.detached): options.project = False project_mode=False else: options.project = True ## output folder print ("\n+ Create output folder(s):") HCGB_files.create_folder(outdir) ### info final_dir = "" if (options.project): print ("+ Generate a directory containing information within the project folder provided") final_dir = HCGB_files.create_subfolder("info", outdir) else: final_dir = outdir ## get files pd_samples_retrieved = sampleParser.files.get_files(options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug) ## Information returned in pd_samples_retrieved ### sample, dirname, name, name_len, lane, read_pair, lane_file, ext, gz if options.debug: HCGB_aes.debug_message("pd_samples_retrieved", "yellow") HCGB_main.print_all_pandaDF(pd_samples_retrieved) ## time stamp start_time_partial = HCGB_time.timestamp(start_time_total) ## check character limitation list_lengths = pd_samples_retrieved.loc[:,'name_len'].to_list() if any(i > 10 for i in list_lengths): print (colored("\t ** Name lengths exceeds the 10 character limitation...", 'yellow')) if not (options.rename): print (colored("** ERROR: Rename files or provide --rename option...", 'red')) exit() ### rename files if (options.rename): options.rename = os.path.abspath(options.rename) if not HCGB_files.is_non_zero_file(options.rename): print (colored("** ERROR: File provided with rename information is not readable.", 'red')) print (options.rename) exit() names_retrieved = pd.read_csv(options.rename, sep=',', index_col=0, squeeze=True, header=None).to_dict() ## read csv to dictionary if (options.debug): HCGB_aes.debug_message("names_retrieved", "yellow") print (names_retrieved) ## TODO: check integrity of new names and special characters ## print to a file timestamp = time_functions.create_human_timestamp() rename_details = final_dir + '/' + timestamp + '_prep_renameDetails.txt' rename_details_hd = open(rename_details, 'w') ## rename files for index, row in pd_samples_retrieved.iterrows(): if (row['gz']): extension_string = row['ext'] + row['gz'] else: extension_string = row['ext'] if options.single_end: renamed = names_retrieved[row['name']] + '.' + extension_string else: renamed = names_retrieved[row['name']] + '_' + row['read_pair'] + '.' + extension_string ## modify frame pd_samples_retrieved.loc[index, 'new_file'] = renamed pd_samples_retrieved.loc[index, 'new_name'] = names_retrieved[row['name']] ## save in file string = row['sample'] + '\t' + renamed + '\n' rename_details_hd.write(string) if (options.debug): print (colored('** DEBUG: rename', 'yellow')) print ("Original: ", row['name']) print ("Renamed: ", names_retrieved[row['name']]) print ("File:", renamed) rename_details_hd.close() ##elif (options.single_end): It should work for both print ("+ Sample files have been renamed...") else: pd_samples_retrieved['new_file'] = pd_samples_retrieved['file'] ## create outdir for each sample outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "raw", options.debug) ## merge option if (options.merge): print ("+ Sample files will be merged...") ## TODO: check when rename option provided pd_samples_merged = sampleParser.merge.one_file_per_sample( pd_samples_retrieved, outdir_dict, options.threads, final_dir, options.debug) if (options.rename): print ("+ Merge files have been renamed...") else: print ("+ Sample files have been merged...") ## process is finished here print ("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print ("+ Exiting prep module.") exit() ## debugging messages if (options.debug): print (colored("** DEBUG: pd_samples_retrieved", 'yellow')) HCGB_main.print_all_pandaDF(pd_samples_retrieved) print (colored("** DEBUG: outdir_dict", 'yellow')) print (outdir_dict) ## copy or create symbolic link for files if (options.copy): print ("+ Sample files will be copied...") ## print to a file timestamp = HCGB_time.create_human_timestamp() copy_details = final_dir + '/' + timestamp + '_prep_copyDetails.txt' copy_details_hd = open(copy_details, 'w') else: print ("+ Sample files will be linked...") list_reads = [] for index, row in pd_samples_retrieved.iterrows(): if (options.copy): ## TODO: debug & set threads to copy faster shutil.copy(row['sample'], os.path.join(outdir_dict[row['new_name']], row['new_file'] )) string = row['sample'] + '\t' + os.path.join(outdir_dict[row['new_name']], row['new_file']) + '\n' copy_details_hd.write(string) else: list_reads.append(row['new_file']) if options.project: HCGB_files.get_symbolic_link_file(row['sample'], os.path.join(outdir_dict[row['new_name']], row['new_file'])) if (options.copy): print ("+ Sample files have been copied...") copy_details_hd.close() else: if not options.project: HCGB_files.get_symbolic_link(list_reads, outdir) print ("\n*************** Finish *******************") start_time_partial = HCGB_time.timestamp(start_time_total) print ("+ Exiting prep module.") return()