def help_edirect():
"""
Prints edirect_caller script help options
"""
HCGB_aes.boxymcboxface("ENTREZ DIRECT (EDirect)")
print("+ Connects with NCBI entrez databases")
print(
"+ Searches, retrieves, and parses data from NCBI databases through the Unix command line"
)
print("+ Provides access to the NCBI's suite of interconnected databases")
print(
"+ Includes an argument-driven function that simplifies the extraction of \ndata from document summaries or other results that are in structured XML format."
)
print("\n")
def run(options):
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Citation")
if (options.option == 'all'):
print(colored("\n+ BacterialTyper citation:", 'blue'))
only_us()
print(colored("\n+ Other softwares employed in the pipeline", 'blue'))
print_all()
elif (options.option == 'only'):
print(colored("\n+ BacterialTyper citation:", 'blue'))
only_us()
return ()
def KMA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases,
time_partial):
"""Kmer identification using software KMA_.
:param options: options passed to the :func:`BacterialTyper.modules.ident.run_ident` main function (threads, KMA_cutoff, etc). See details in...
:param pd_samples_retrieved: pandas dataframe for samples to process.
:param outdir_dict: dictionary containing information for each sample of the output folder for this process.
:param retrieve_databases:
:param time_partial: timestamp of start time of the process.
:type options:
:type pd_samples_retrieved: pandas.DataFrame()
:type outdir_dict: Dictionary
:type retrieve_databases: pandas.DataFrame()
:type time_partial:
:return: Information of the identification. See example below.
:rtype: pandas.DataFrame()
See example of returned dataframe in file :file:`/devel/results/KMA_ident_example.csv` here:
.. include:: ../../devel/results/KMA_ident_example.csv
:literal:
.. seealso:: This function depends on other ``BacterialTyper`` functions called:
- :func:`BacterialTyper.config.set_config.get_exe`
- :func:`BacterialTyper.scripts.functions.boxymcboxface`
- :func:`BacterialTyper.modules.ident.send_kma_job`
- :func:`BacterialTyper.modules.ident.get_outfile`
- :func:`BacterialTyper.scripts.species_identification_KMA.check_db_indexed`
- :func:`BacterialTyper.scripts.species_identification_KMA.parse_kma_results`
.. include:: ../../links.inc
"""
return (pd.DataFrame())
### print header
HCGB_aes.boxymcboxface("KMA Identification")
## set defaults
kma_bin = set_config.get_exe("kma")
## check status
databases2use = []
for index, db2use in retrieve_databases.iterrows():
## index_name
if (str(db2use['source']).startswith('KMA')):
print('+ Check database: ' + db2use['db'])
fold_name = os.path.dirname(db2use['path'])
index_status = species_identification_KMA.check_db_indexed(
db2use['path'], fold_name)
if (index_status == True):
print(
colored(
"\t+ Databases %s seems to be fine...\n\n" %
db2use['db'], 'green'))
databases2use.append(db2use['path'])
else:
#databases2use.remove(db2use)
print(
colored(
"\t**Databases %s is not correctly indexed. Not using it...\n"
% db2use['db'], 'red'))
## debug message
if (Debug):
print(
colored(
"**DEBUG: databases2use\n" + "\n".join(databases2use) + "\n**",
'yellow'))
## Start identification of samples
print("\n+ Send KMA identification jobs...")
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
## send for each sample
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
for db2use in databases2use:
## load database on memory
print("+ Loading database on memory for faster identification.")
return_code_load = species_identification_KMA.load_db(
kma_bin, db2use)
## send for each sample
commandsSent = {
executor.submit(send_kma_job, outdir_dict[name],
sorted(cluster["sample"].tolist()), name,
db2use, threads_job, Debug): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
## remove database from memory
print("+ Removing database from memory...")
return_code_rm = species_identification_KMA.remove_db(
kma_bin, db2use)
if (return_code_rm == 'FAIL'):
print(
colored(
"***ERROR: Removing database from memory failed. Please do it manually! Execute command: %s"
% cmd_rm_db, 'red'))
## functions.timestamp
time_partial = HCGB_time.timestamp(time_partial)
## parse results
print("+ KMA identification call finished for all samples...")
print("+ Parse results now")
results_summary = pd.DataFrame()
for db2use in databases2use:
### [TODO]: parse data according to database: bacteria, plasmids or user data or genbank data provided
basename_db = os.path.basename(db2use)
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
###
for name, cluster in sample_frame:
## get result
## outdir_KMA
outdir_dict_kma = HCGB_files.create_subfolder(
"kma", outdir_dict[name])
result = get_outfile(outdir_dict_kma, name, db2use)
#print ('\t- File: ' + result + '.spa')
## get results using a cutoff value [Defaulta: 80]
results = species_identification_KMA.parse_kma_results(
result + '.spa', options.KMA_cutoff)
results['Database'] = basename_db
### check if db2use is plasmids as it could be several.
if (results.index.size > 1):
if (basename_db == "plasmids.T" or basename_db == "viral.TG"):
## let it be several entries
results['Sample'] = name
results_summary = results_summary.append(results,
ignore_index=True)
else:
print(
colored("###########################################",
'yellow'))
print(
colored("Sample %s contains multiple strains." % name,
'yellow'))
print(
colored("###########################################",
'yellow'))
print(colored(results, 'yellow'))
print('\n\n')
## add both strains if detected
results['Sample'] = name
results_summary = results_summary.append(results,
ignore_index=True)
## TODO: add multi-isolate flag
elif (results.index.size == 1): ## 1 clear reference
results['Sample'] = name
results_summary = results_summary.append(results,
ignore_index=True)
else:
print(
colored(
'\tNo clear strain from database %s has been assigned to sample %s'
% (basename_db, name), 'yellow'))
## add empty line if no available
results['Sample'] = name
results_summary = results_summary.append(results,
ignore_index=True)
print("+ Finish this step...")
## debug message
if (Debug):
results_summary.to_csv(quotechar='"')
return (results_summary)
def run_input(arg_dict):
"""Main function of the input_parser module in BacDup package.
This module prepares data for later gene duplication analysis.
It allows the user to provide either a single sample, multiple samples, NCBI
GenBank IDs or NCBI taxonomy IDs to retrieve and obtain the annotation data.
"""
## help message
if (arg_dict.input_help):
help_input()
exit()
BacDup_functions.pipeline_header('BacDup')
HCGB_aes.boxymcboxface("Preparing input files")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## init time
start_time_total = time.time()
## absolute path for in & out
#input_dir = os.path.abspath(options.input)
outdir = os.path.abspath(arg_dict.output_folder)
## output folder
print("\n+ Create output folder(s):")
HCGB_files.create_folder(outdir)
## set defaults
if not (arg_dict.assembly_level):
arg_dict.assembly_level = 'complete'
if not (arg_dict.section):
arg_dict.section = 'genbank'
## project or detached?
if arg_dict.detached:
arg_dict.project = False
final_dir = outdir
data_dir = outdir
else:
arg_dict.project = True
print(
"+ Generate a directory containing information within the project folder provided"
)
final_dir = HCGB_files.create_subfolder("info", outdir)
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Project/Detached option:', 'yellow')
debug_message('arg_dict.detached: ' + str(arg_dict.detached), 'yellow')
debug_message('arg_dict.project: ' + str(arg_dict.project), 'yellow')
debug_message('outdir:' + outdir, 'yellow')
debug_message('final_dir:' + final_dir, 'yellow')
debug_message('+++++++++++++++++++++++++++++++')
## get files
print()
HCGB_aes.print_sepLine("-", 50, False)
print('+ Getting input information provided... ')
print('+ Several options available:')
print('\t* Single/Multiple Annotation file:')
print('\t |-- GenBank format files')
print('\t |-- GFF files + Reference fasta files required')
print('\n\t* Single/Multiple NCBI GenBank IDs')
print('\n\t* Single/Multiple NCBI taxonomy IDs + Options')
print('\n\t* A previous BacDup project folder')
print('\n+ Check the option provided...')
time.sleep(1)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
#################################################
## Parse and obtain the type of input information provided
#################################################
df_accID = parse_options(arg_dict)
## pd.DataFrame: 'new_name','folder','genus',
## 'species','taxonomy','genome',
## 'annot_file','format_annot_file', 'proteins',
## 'plasmids_number','plasmids_ID'))
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## parse information accordingly
parse_information(arg_dict, df_accID, outdir)
### report generation
HCGB_aes.boxymcboxface("Summarizing input files")
outdir_report = HCGB_files.create_subfolder("report", outdir)
input_report = HCGB_files.create_subfolder("input", outdir_report)
## add df_accID.loc[sample,] information as csv into input folder
df_accID.to_csv(os.path.join(input_report, 'info.csv'),
index=True,
header=True)
## maybe add a summary of the files?
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Input module.")
return ()
def fastqc(pd_samples_retrieved, outdir, options, start_time_total,
name_analysis, Debug):
HCGB_aes.boxymcboxface("FASTQC Quality check for samples")
## debug message
if (Debug):
print(colored("\n**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
print("\n")
## generate output folder, if necessary
print("\n+ Create output folder(s):")
## if not project, outdir contains the dir to put output
## in this case, in some other cases might not occur
if not options.project:
functions.create_folder(outdir)
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved,
"fastqc_" + name_analysis,
options.debug)
print("+ Checking quality for each sample retrieved...")
start_time_partial = start_time_total
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
HCGB_aes.debug_message("options.threads: " + str(options.threads),
"yellow")
HCGB_aes.debug_message("max_workers: " + str(max_workers_int),
"yellow")
HCGB_aes.debug_message("threads_job: " + str(threads_job), "yellow")
## send for each sample
print("+ Calling fastqc for samples...")
with concurrent.futures.ThreadPoolExecutor(
max_workers=int(max_workers_int)) as executor:
commandsSent = {
executor.submit(fastqc_caller.run_module_fastqc, outdir_dict[name],
sorted(cluster["sample"].tolist()), name,
threads_job): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
print("+ FASTQC for samples has finished...")
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
if (options.skip_report):
print("+ No report generation...")
else:
print("\n+ Generating a report using MultiQC module.")
outdir_report = HCGB_files.create_subfolder("report", outdir)
## get subdirs generated and call multiQC report module
givenList = []
print(
"+ Detail information for each sample could be identified in separate folders:"
)
## call multiQC report module
givenList = [v for v in outdir_dict.values()]
my_outdir_list = set(givenList)
## debug message
if (Debug):
print(
colored("\n**DEBUG: my_outdir_list for multiqc report **",
'yellow'))
print(my_outdir_list)
print("\n")
fastqc_report = HCGB_files.create_subfolder("FASTQC", outdir_report)
fastqc_final_report = HCGB_files.create_subfolder(
name_analysis, fastqc_report)
multiQC_report.multiQC_module_call(my_outdir_list, "FASTQC",
fastqc_final_report, "")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% fastqc_final_report)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting qc module.")
exit()
def run_profile(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
if (options.help_project):
## information for project
help_info.project_help()
exit()
if (options.help_ARIBA):
## help_format option
ariba_caller.help_ARIBA()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Virulence & Resistance profile module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
options.database = os.path.abspath(options.database)
global input_dir
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
global Project
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "profile",
options.debug)
###
print(
"+ Generate a sample profile for virulence and resistance candidate genes for each sample retrieved using:"
)
print(
"(1) Antimicrobial Resistance Inference By Assembly (ARIBA) software")
print(
"(2) Pre-defined databases by different suppliers or user-defined databases."
)
## get databases to check
retrieve_databases = get_options_db(options)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_total)
########
ARIBA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases,
start_time_partial)
######################################
## update database for later usage
######################################
if not options.fast:
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
HCGB_aes.boxymcboxface("Update Sample Database")
## update db
print("+ Update database with samples identified")
## TODO: check if it works
dataBase_user = database_user.update_database_user_data(
options.database, input_dir, Debug, options)
## debug message
if (Debug):
print(colored("**DEBUG: results obtained **", 'yellow'))
else:
print(
"+ No update of the database has been requested using option --fast"
)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Virulence & Resistance profile module.")
return ()
def run_search(arg_dict):
"""Main function of the search module in BacDup package.
This module searches and create gene duplication analysis.
It allows the user to provide either a previous parsed data project (NCBI Genbank IDs, taxonomy or user
annotation data) or a single or multiple samples.
"""
## help message
if (arg_dict.input_help):
help_input()
exit()
if (arg_dict.blast_help):
info.blast_help()
exit()
if (arg_dict.project_help):
info.project_help()
exit()
if (arg_dict.detached_mode_help):
info.detached_mode()
exit()
### Start the analysis
BacDup_functions.pipeline_header('BacDup')
HCGB_aes.boxymcboxface("Search module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## init time
start_time_total = time.time()
## absolute path for in & out
outdir = os.path.abspath(arg_dict.input_folder)
## project or detached?
if arg_dict.detached:
arg_dict.project = False
## output folder
print("\n+ Create output folder(s):")
HCGB.functions.files_functions.create_folder(outdir)
else:
arg_dict.project = True
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Project/Detached option:', 'yellow')
debug_message('arg_dict.detached: ' + str(arg_dict.detached), 'yellow')
debug_message('arg_dict.project: ' + str(arg_dict.project), 'yellow')
debug_message('outdir:' + outdir, 'yellow')
debug_message('+++++++++++++++++++++++++++++++')
## get files
print()
HCGB_aes.print_sepLine("-", 50, False)
print('+ Getting information provided... ')
print('+ Several options available:')
print('\t* BacDup project folder with initiated data')
print('\t* Single/Multiple Annotation file:')
print('\t |-- GenBank format files')
print('\t |-- GFF files + Reference fasta files required')
print('\t* Single/Multiple raw BLAST results files')
print('\t* Single/Multiple fasta proteins + annotation table')
print("""\n\n**** NOTE: ****
For additional options (e.g. Single/Multiple NCBI GenBank or taxonomy IDs)
use the input module to accommodate accordingly """)
time.sleep(1)
print()
## parse options
pd_samples_retrieved = parse_search_options(arg_dict)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## for each sample
dict_search_folders = HCGB.functions.files_functions.outdir_project(
outdir, arg_dict.project, pd_samples_retrieved, "search",
arg_dict.debug)
dict_dup_folders = HCGB.functions.files_functions.outdir_project(
outdir, arg_dict.project, pd_samples_retrieved, "dups", arg_dict.debug)
dict_parse_folders = HCGB.functions.files_functions.outdir_project(
outdir, arg_dict.project, pd_samples_retrieved, "parse",
arg_dict.debug)
## create results
data2add = pd.DataFrame(columns=BacDup_functions.columns_dup_table())
for sample, folder in dict_search_folders.items():
annot_timestamp = os.path.join(dict_dup_folders[sample],
'.annot_success')
dup_annot_file = os.path.join(dict_dup_folders[sample],
'dup_annot.csv')
## annotation
annot_table_file = pd_samples_retrieved.loc[sample, 'annot_table']
if (not HCGB.functions.files_functions.is_non_zero_file(
annot_timestamp)):
## get results
file_data = pd_samples_retrieved.loc[sample, 'file_data']
format = pd_samples_retrieved.loc[sample, 'format']
filtered_data = dup_searcher.filter_data(
sample, file_data, format, arg_dict.pident, arg_dict.evalue,
arg_dict.percentage, arg_dict.bitscore, folder, arg_dict.debug)
## timestamps
filter_timestamp = os.path.join(dict_dup_folders[sample],
'.filter_success')
if (not HCGB.functions.files_functions.is_non_zero_file(
filter_timestamp)):
#save results as a .csv file
sort_csv = os.path.abspath(
os.path.join(dict_dup_folders[sample],
'filtered_results.csv'))
filtered_data.to_csv(sort_csv, header=True, index=False)
## print time stamp
HCGB_time.print_time_stamp(filter_timestamp)
else:
read_time = HCGB_time.read_time_stamp(filter_timestamp)
print(
colored(
"\t+ Filter results already available for sample %s [%s]"
% (sample, read_time), 'green'))
## get annotation
(dup_annot_df, data2add_entry) = dup_searcher.get_dupannot(
sample, filtered_data, annot_table_file, arg_dict.debug)
##
info_dup_file = os.path.join(dict_dup_folders[sample],
'info_dup.csv')
data2add_entry.to_csv(info_dup_file, header=True, index=False)
## save into file
dup_annot_df.to_csv(dup_annot_file, header=True)
## print time stamp
HCGB_time.print_time_stamp(annot_timestamp)
else:
read_time = HCGB_time.read_time_stamp(annot_timestamp)
print(
colored(
"\t+ Duplicate annotation already available for sample %s [%s]"
% (sample, read_time), 'green'))
## add info for each
dup_annot_df = HCGB_main.get_data(dup_annot_file, ',',
"index_col=0")
annot_table = HCGB_main.get_data(annot_table_file, ',',
"index_col=0")
data2add_entry = dup_searcher.get_dup_stats(
sample, dup_annot_df, annot_table, arg_dict.debug)
## add genome length data
data2add_entry['genome_len'] = ''
len_df_file = os.path.join(dict_parse_folders[sample], 'length_df.csv')
if os.path.isfile(len_df_file):
len_data = HCGB_main.get_data(len_df_file, ',', "header=None")
data2add_entry['genome_len'] = len_data[1].sum()
## merge data
#data2add_entry = data2add_entry.reset_index()
data2add = data2add.append(data2add_entry, ignore_index=False)
### report generation
HCGB_aes.boxymcboxface("Summarizing duplicated search")
outdir_report = HCGB.functions.files_functions.create_subfolder(
"report", outdir)
dups_report = HCGB.functions.files_functions.create_subfolder(
"dups", outdir_report)
## add data2add
data2add.to_csv(os.path.join(dups_report, 'info_annot.csv'),
index=True,
header=True)
## maybe add a summary of the files?
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting search module.")
return ()
def MLST_ident(options, dataFrame, outdir_dict, dataFrame_edirect,
retrieve_databases):
"""Generate MLST profile identification
This functions uses the `MLSTar software`_ to retrieve Multi locus sequence typing (MLST) profiles from PubMLST_ for the given species previously identified by KMA. It generates MLST profiling for each sample.
:param options: options passed to the :func:`BacterialTyper.modules.ident.run_ident` main function (threads, KMA_cutoff, etc). See details in...
:param dataFrame: pandas dataframe for samples to process. Result from :func:`BacterialTyper.modules.ident.KMA_ident`.
:param outdir_dict: dictionary containing information for each sample of the output folder for this process.
:param dataFrame_edirect: pandas dataframe resulted from :func:`BacterialTyper.modules.ident.edirect_ident`.
:param retrieve_databases:
:type options:
:type dataFrame: pandas.DataFrame()
:type outdir_dict: Dictionary
:type dataFrame_edirect: pandas.DataFrame()
:type retrieve_databases: pandas.DataFrame()
:return: Information of the MLST identification. Dictionary keys are samples and values are the absolute path to file generate by :func:`BacterialTyper.scripts.MLSTar.run_doMLST` containing MLST information.
:rtype: Dictionary
See example of returned dataframe in file :file:`/devel/results/doMLST_result_example.csv` here:
.. include:: ../../devel/results/doMLST_result_example.csv
:literal:
.. seealso:: Additional information to PubMLST available datasets.
- :doc:`PubMLST datasets<../../../data/PubMLST_datasets>`
.. seealso:: This function depends on other ``BacterialTyper`` functions called:
- :func:`BacterialTyper.scripts.functions.read_time_stamp`
- :func:`BacterialTyper.scripts.functions.create_subfolder`
- :func:`BacterialTyper.scripts.functions.boxymcboxface`
- :func:`BacterialTyper.scripts.MLSTar.run_MLSTar`
- :func:`HCGB.sampleParser.files.get_files`
- :func:`BacterialTyper.scripts.MLSTar.get_MLSTar_species`
.. include:: ../../links.inc
"""
## set config
rscript = set_config.get_exe("Rscript")
## TODO: Samples might not be assembled...to take into account and return 0
## TODO: Fix and install MLSTar during installation
print(MLSTar.get_MLSTar_package_installed())
exit()
########################################################################################
## TODO: What to do if multi-isolate sample?
## TODO: Control if a different profile is provided via --MLST_profile
## TODO: Check time passed and download again if >?? days passed]
## debug message
if (Debug):
print(colored("**DEBUG: dataFrame_edirect identified**", 'yellow'))
print(dataFrame_edirect)
## MLST call
HCGB_aes.boxymcboxface("MLST typing")
print(
"+ Create classical MLST typification of each sample according to species retrieved by kmer..."
)
## get assembly files
input_dir = os.path.abspath(options.input)
assembly_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: assembly_samples_retrieved**", 'yellow'))
print(assembly_samples_retrieved)
# init
MLST_results = {}
## get MLST_profile: default or provided
mlst_profile_list = retrieve_databases.loc[retrieve_databases['db'] ==
'PubMLST']['path'].tolist()
if (Debug):
print("** Debug **")
print("mlst_profile_list")
print(mlst_profile_list)
print("dataFrame_edirect")
print(dataFrame_edirect)
## Generate MLST call according to species identified for each sample
for index, row in dataFrame_edirect.iterrows():
MLSTar_taxa_name = MLSTar.get_MLSTar_species(row['genus'],
row['species'])
if (MLSTar_taxa_name == 'NaN'):
print(
colored(
"\t- Not available PubMLST profile for sample [%s] identified as %s %s"
% (row['sample'], row['genus'], row['species']), 'yellow'))
else:
for mlst_profile in mlst_profile_list:
## species folder
#species_mlst_folder = functions.create_subfolder(MLSTar_taxa_name, pubmlst_folder)
species_mlst = mlst_profile.split(',')[0]
species_mlst_folder = mlst_profile.split(',')[1]
## output file
output_file = species_mlst_folder + '/PubMLST_available_scheme.csv'
filename_stamp = species_mlst_folder + '/.success_scheme'
##
if MLSTar_taxa_name == species_mlst:
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print(
colored(
"\tA previous command generated results on: %s"
% stamp, 'yellow'))
else:
### get scheme available
MLSTar.getPUBMLST(MLSTar_taxa_name, rscript,
output_file)
stamp = HCGB_time.print_time_stamp(filename_stamp)
## parse and get scheme for classical MLST
schemes_MLST = pd.read_csv(output_file, sep=',', header=0)
##
for item, cluster in schemes_MLST.iterrows():
if cluster['len'] < 10:
scheme2use = int(cluster['scheme'])
continue
###
sample = row['sample']
MLSTar_folder = HCGB_files.create_subfolder(
'MLST', outdir_dict[sample])
genome_file = assembly_samples_retrieved.loc[
assembly_samples_retrieved['name'] ==
sample]['sample'].values[0]
## call MLST
(results, profile_folder) = MLSTar.run_MLSTar(
species_mlst_folder, rscript, MLSTar_taxa_name,
scheme2use, sample, MLSTar_folder, genome_file,
options.threads)
MLST_results[sample] = results
##
print("+ Finish this step...")
return (MLST_results)
def run_annotation(options):
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
elif (options.help_Prokka):
## information for Prokka
annotation.print_list_prokka()
exit()
## set default
options.batch = False
###
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Assembly annotation")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
### symbolic links
print("+ Retrieve all genomes assembled...")
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for samples
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "annot",
options.debug)
## annotate
print("+ Annotate assemblies using prokka:")
print("\t-Option: kingdom = ", options.kingdom, "; Annotation mode")
if options.genera == 'Other':
print(
"\t-Option: genera = Off; No genus-specific BLAST databases option provided"
)
else:
print("\t-Option: genera = ", options.genera,
"; Genus-specific BLAST databases option provided")
print("\t-Option: addgenes; Add 'gene' features for each 'CDS' feature")
print("\t-Option: addmrna; Add 'mRNA' features for each 'CDS' feature")
print("\t-Option: cdsrnaolap; Allow [tr]RNA to overlap CDS")
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
## send for each sample
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
commandsSent = {
executor.submit(annot_caller, row['sample'],
outdir_dict[row['name']], options, row['name'],
threads_job): index
for index, row in pd_samples_retrieved.iterrows()
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## get folders
givenList = [v for v in outdir_dict.values()]
protein_files = []
print(
"+ Detail information for each sample could be identified in separate folders:"
)
for folder in givenList:
print('\t + ', folder)
protein_files.extend(
HCGB_main.retrieve_matching_files(folder, '.faa', Debug))
### report generation
if (options.skip_report):
print("+ No annotation report generation...")
else:
### report generation
HCGB_aes.boxymcboxface("Annotation report")
outdir_report = HCGB_files.create_subfolder("report", outdir)
PROKKA_report = HCGB_files.create_subfolder("annotation",
outdir_report)
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% PROKKA_report)
## check if previously report generated
filename_stamp = PROKKA_report + '/.success'
done = 0
if os.path.isdir(PROKKA_report):
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print(
colored(
"\tA previous report generated results on: %s" % stamp,
'yellow'))
done = 1
## generate report
if done == 0:
## get subdirs generated and call multiQC report module
multiQC_report.multiQC_module_call(givenList, "Prokka",
PROKKA_report, "-dd 2")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% PROKKA_report)
## success stamps
filename_stamp = PROKKA_report + '/.success'
stamp = HCGB_time.print_time_stamp(filename_stamp)
## time stamp
start_time_partial_BUSCO = HCGB_time.timestamp(start_time_total)
## Check each annotation using BUSCO
results = qc.BUSCO_check(input_dir, outdir, options,
start_time_partial_BUSCO, "proteins")
## print to file: results
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Annotation module.")
return ()
def run(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_trimm_adapters):
## help on trimm adapters
trimmomatic_call.print_help_adapters()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Trimming samples")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"),
options.debug)
## debug message
if (Debug):
HCGB_aes.debug_message("pd_samples_retrieved", 'yellow')
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for samples
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "trimm",
options.debug)
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
print("+ Trimming adapters for each sample retrieved...")
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
# Trimming adapters
if (options.adapters):
# Adapter file provided
options.adapters = os.path.abspath(options.adapters)
print("\t- Adapters file provided...")
else:
# Get default adpaters file
print("\t- Default Trimmomatic adapters (v0.39) will be used...")
options.adapters = data_files.data_list(
"available_Trimmomatic_adapters")
## send for each sample
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
commandsSent = {
executor.submit(trimmo_caller, sorted(cluster["sample"].tolist()),
outdir_dict[name], name, threads_job, Debug,
options.adapters): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
print("\n\n+ Trimming samples has finished...")
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## get files generated and generate symbolic link
if not options.project:
dir_symlinks = HCGB_files.create_subfolder('link_files', outdir)
files2symbolic = []
folders = os.listdir(outdir)
## debug message
if (Debug):
print(
colored(
"**DEBUG: generate symbolic links for each file in " +
dir_symlinks + "**", 'yellow'))
for fold in folders:
if fold.endswith(".log"):
continue
else:
this_folder = outdir + '/' + fold
subfiles = os.listdir(this_folder)
for files in subfiles:
files_search = re.search(
r".*trim_R\d{1}.*",
files) ## only paired-end. Todo: single end
if files_search:
files2symbolic.append(this_folder + '/' + files)
HCGB_files.get_symbolic_link(files2symbolic, dir_symlinks)
if (options.skip_report):
print("+ No report generation...")
else:
print("\n+ Generating a report using MultiQC module.")
outdir_report = HCGB_files.create_subfolder("report", outdir)
## call multiQC report module
givenList = [v for v in outdir_dict.values()]
my_outdir_list = set(givenList)
## debug message
if (Debug):
HCGB_aes.debug_message("my_outdir_list for multiqc report",
"yellow")
print(my_outdir_list)
print("\n")
trimm_report = HCGB_files.create_subfolder("trimm", outdir_report)
multiQC_report.multiQC_module_call(my_outdir_list, "Trimmomatic",
trimm_report, "")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% trimm_report)
## create fastqc for trimmed reads
pd_samples_retrieved_trimmed = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
qc.fastqc(pd_samples_retrieved_trimmed, outdir, options,
start_time_partial, "trimmed", Debug)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("\n+ Exiting trimm module.")
return ()
def run_biotype(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_XICRA.help_fastq_format()
elif (options.help_project):
## information for project
help_XICRA.project_help()
exit()
elif (options.help_RNAbiotype):
## information for join reads
RNAbiotype.help_info()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
aesthetics_functions.pipeline_header('XICRA')
aesthetics_functions.boxymcboxface("RNA biotype analysis")
print("--------- Starting Process ---------")
time_functions.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
if (options.detached):
outdir = os.path.abspath(options.output_folder)
options.project = False
else:
options.project = True
outdir = input_dir
## get files
print('+ Getting files from input folder... ')
## get files
if options.noTrim:
print('+ Mode: fastq.\n+ Extension: ')
print("[ fastq, fq, fastq.gz, fq.gz ]\n")
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"),
options.debug)
else:
print('+ Mode: trim.\n+ Extension: ')
print("[ _trim_ ]\n")
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## Discard if joined reads: use trimmed single-end or paired-end
pd_samples_retrieved = pd_samples_retrieved[
pd_samples_retrieved['ext'] != '_joined']
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
files_functions.create_folder(outdir)
## for samples
mapping_outdir_dict = files_functions.outdir_project(
outdir, options.project, pd_samples_retrieved, "map", options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: mapping_outdir_dict **", 'yellow'))
print(mapping_outdir_dict)
# time stamp
start_time_partial = time_functions.timestamp(start_time_total)
## optimize threads
name_list = set(pd_samples_retrieved["new_name"].tolist())
threads_job = main_functions.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
##############################################
## map Reads
##############################################
start_time_partial = mapReads_module(options, pd_samples_retrieved,
mapping_outdir_dict, options.debug,
max_workers_int, threads_job,
start_time_partial, outdir)
## debug message
if (Debug):
print(colored("**DEBUG: mapping_results **", 'yellow'))
print(mapping_results)
# time stamp
start_time_partial = time_functions.timestamp(start_time_partial)
## for samples
biotype_outdir_dict = files_functions.outdir_project(
outdir, options.project, pd_samples_retrieved, "biotype",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: biotype_outdir_dict **", 'yellow'))
print(biotype_outdir_dict)
## get RNAbiotype information
RNAbiotype.RNAbiotype_module_call(mapping_results, biotype_outdir_dict,
options.annotation, options.debug,
max_workers_int, threads_job)
# time stamp
start_time_partial = time_functions.timestamp(start_time_partial)
if (options.skip_report):
print("+ No report generation...")
else:
print(
"\n+ Generating a report using MultiQC module for featureCount analysis."
)
outdir_report = files_functions.create_subfolder("report", outdir)
## get subdirs generated and call multiQC report module
givenList = []
print(
"+ Detail information for each sample could be identified in separate folders:"
)
## call multiQC report module
givenList = [v for v in biotype_outdir_dict.values()]
my_outdir_list = set(givenList)
## debug message
if (Debug):
print(
colored("\n**DEBUG: my_outdir_list for multiqc report **",
'yellow'))
print(my_outdir_list)
print("\n")
featureCount_report = files_functions.create_subfolder(
"featureCount", outdir_report)
multiQC_report.multiQC_module_call(my_outdir_list, "featureCount",
featureCount_report, "-dd 2")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% featureCount_report)
### Summarizing RNA biotype information
biotype_report = files_functions.create_subfolder(
"biotype", outdir_report)
single_files_biotype = files_functions.create_subfolder(
"samples", biotype_report)
## results
dict_files = {}
for samples in biotype_outdir_dict:
featurecount_file = os.path.join(biotype_outdir_dict[samples],
'featureCount.out.tsv')
if files_functions.is_non_zero_file(featurecount_file):
dict_files[samples] = featurecount_file
## copy pdf
pdf_plot = main_functions.retrieve_matching_files(
biotype_outdir_dict[samples], '.pdf', options.debug)
if files_functions.is_non_zero_file(pdf_plot[0]):
shutil.copy(pdf_plot[0], single_files_biotype)
## collapse all information
all_data = RNAbiotype.generate_matrix(dict_files)
## print into excel/csv
print('+ Table contains: ', len(all_data), ' entries\n')
## debugging messages
if Debug:
print("** DEBUG: all_data")
print(all_data)
## set abs_csv_outfile to be in report folder
## copy or link files for each sample analyzed
abs_csv_outfile = os.path.join(biotype_report, "summary.csv")
all_data.to_csv(abs_csv_outfile)
## create plot: call R [TODO: implement in python]
outfile_pdf = os.path.join(biotype_report, "RNAbiotypes_summary.pdf")
## R scripts
biotype_R_script = tools.R_scripts('plot_RNAbiotype_sum',
options.debug)
rscript = set_config.get_exe("Rscript", options.debug)
cmd_R_plot = "%s %s -f %s -o %s" % (rscript, biotype_R_script,
abs_csv_outfile, outfile_pdf)
##
print("+ Create summary plot for all samples")
callCode = system_call_functions.system_call(cmd_R_plot)
print("\n*************** Finish *******************")
start_time_partial = time_functions.timestamp(start_time_total)
print("\n+ Exiting join module.")
return ()
def run(options):
"""
This is the main function of the module ``config``. It basically checks
if the different requirements (python` and third-party software) are
fulfilled.
If any requirement is not available this modules tries to install them or reports to the user to
manually install them.
:param option: State whether to check or install missing modules, packages and third party software. Provide: check/install
:param install_path: Absolute path to install modules or packages missing. Default: ``BacterialTyper`` environment path.
:param IslandPath: True/False for checking additional perl and software required by this option analysis.
:param debug: True/false for debugging messages.
:type option: string
:type IslandPath: boolean
:type install_path: string
:type debug: boolean
.. seealso:: This function depends on several ``BacterialTyper`` functions:
- :func:`BacterialTyper.config.set_config.check_python_packages`
- :func:`BacterialTyper.config.set_config.check_perl_packages`
- :func:`BacterialTyper.config.extern_progs.return_min_version_soft`
- :func:`BacterialTyper.config.extern_progs.print_dependencies`
"""
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Pipeline Configuration")
print("--------- Starting Process ---------")
HCGB_time.print_time()
if (options.install_path):
if os.path.isdir(options.install_path):
if (Debug):
print(
"Installation path provided for missing modules, packages, dependencies..."
)
print("Path: " + options.install_path)
else:
print(colored("\n*** ERROR ****", 'red'))
print(colored("Path provided is not a folder", 'red'))
print(options.install_path)
exit()
else:
## get python environment path
env_bin_directory = os.path.dirname(os.environ['_'])
##os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'templates'))
options.install_path = os.path.abspath(
os.path.join(env_bin_directory, '../software'))
if (Debug):
print("Retrieve environment path as installation path:")
print("Path: " + options.install_path)
HCGB_files.create_folder(options.install_path)
#######################
## install or only check
#######################
option_install = False
if (options.option == 'install'):
print("\n+ Check dependencies")
print(
"+ Try to install all missing dependencies, modules or third party software..."
)
option_install = True
## check if access and permission
if os.path.isdir(options.install_path):
if (set_config.access_check(options.install_path, mode=os.F_OK)):
print(
"Installation path is accessible and has permission for installation if necessary"
)
else:
print(colored("\n*** ERROR ****", 'red'))
print(
colored(
"No access/permission for this path: %s" %
options.install_path, 'red'))
print(
colored(
"Please provide a valid path with access/permission to install any missing dependencies.",
'red'))
exit()
else:
print(colored("\n*** ERROR ****", 'red'))
print(colored("Path provided is not a folder", 'red'))
print(options.install_path)
exit()
elif (options.option == 'only_check'):
print(
"\nCheck dependencies, modules or third party software and print report..."
)
#######################
## python version
#######################
HCGB_aes.print_sepLine("+", 20, False)
print('Python:')
HCGB_aes.print_sepLine("+", 20, False)
this_python_version = str(sys.version)
python_min_version = extern_progs.return_min_version_soft('python')
if LooseVersion(this_python_version) >= LooseVersion(python_min_version):
print(
colored(
"Minimum version (%s) satisfied: %s" %
(python_min_version, this_python_version), 'green'))
else:
print(
colored(
"Minimum version (%s) not satisfied: %s" %
(python_min_version, this_python_version), 'red'))
exit()
#######################
## perl_version
#######################
print('\n')
HCGB_aes.print_sepLine("+", 50, False)
print('Perl:')
HCGB_aes.print_sepLine("+", 50, False)
perl_min_version = extern_progs.return_min_version_soft('perl')
this_perl_path = set_config.get_exe("perl", Debug)
this_perl_version = set_config.get_version("perl", this_perl_path, Debug)
if LooseVersion(this_perl_version) >= LooseVersion(perl_min_version):
print(
colored(
"Minimum version (%s) satisfied: %s" %
(perl_min_version, this_perl_version), 'green'))
else:
print(
colored(
"Minimum version (%s) not satisfied: %s" %
(perl_min_version, this_perl_version), 'red'))
exit()
#######################
## third-party software
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('External dependencies:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_dependencies(option_install, options.install_path, Debug)
print('\n')
#######################
## python packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('Python packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_python_packages(Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## perl packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('Perl packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_perl_packages("perl_dependencies", Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## IslandPath dependencies
#######################
if (options.IslandPath):
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('IslandPath packages and software required:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_IslandPath(Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## R packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('R packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_R_packages(option_install, options.install_path, Debug)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
def run_QC(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## set main header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Quality check")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
### option
if (options.raw_reads):
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"),
options.debug)
fastqc(pd_samples_retrieved, outdir, options, start_time_total, "raw",
Debug)
elif (options.trim_reads):
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
fastqc(pd_samples_retrieved, outdir, options, start_time_total,
"trimmed", Debug)
elif (options.assembly):
BUSCO_check(input_dir, outdir, options, start_time_total, "genome")
elif (options.annotation):
BUSCO_check(input_dir, outdir, options, start_time_total, "proteins")
return ()
def BUSCO_check(input_dir, outdir, options, start_time_total, mode):
HCGB_aes.boxymcboxface("BUSCO Analysis Quality check")
## absolute path for in & out
database_folder = os.path.abspath(options.database)
## get files and get dir for each sample according to mode
if mode == 'genome':
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
if not options.project:
outdir = HCGB_files.create_subfolder("assembly_qc", outdir)
if options.debug:
print("** DEBUG: pd_samples_retrieved")
print(pd_samples_retrieved)
BUSCO_outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved,
"assemble_qc",
options.debug)
elif mode == 'proteins':
pd_samples_retrieved = sampleParser.files.get_files(
options, outdir, "annot", ["faa"], options.debug) ##
if not options.project:
outdir = HCGB_files.create_subfolder("annot_qc", outdir)
if options.debug:
print("** DEBUG: pd_samples_retrieved")
print(pd_samples_retrieved)
BUSCO_outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved,
"annot_qc",
options.debug)
## add column to dataframe
pd_samples_retrieved['busco_folder'] = ""
for index, row in pd_samples_retrieved.iterrows():
pd_samples_retrieved.at[index, 'busco_folder'] = BUSCO_outdir_dict[
row['name']]
## debug message
if (options.debug):
HCGB_aes.debug_message("df_samples_busco", 'yellow')
print(pd_samples_retrieved)
HCGB_aes.debug_message("BUSCO_outdir_dict", 'yellow')
print(BUSCO_outdir_dict)
## Check each using BUSCO
database_folder = os.path.abspath(options.database)
BUSCO_Database = HCGB_files.create_subfolder('BUSCO', database_folder)
if not os.path.exists(BUSCO_Database):
HCGB_files.create_folder(BUSCO_Database)
## call
(dataFrame_results, stats_results) = BUSCO_caller.BUSCO_call(
options.BUSCO_dbs, pd_samples_retrieved, BUSCO_Database,
options.threads, mode)
## debug message
if (options.debug):
HCGB_aes.debug_message("dataFrame_results", 'yellow')
HCGB_main.print_all_pandaDF(dataFrame_results)
## functions.timestamp
print("+ Quality control of all samples finished: ")
start_time_partial = HCGB_time.timestamp(start_time_total)
## multiqc report plot
if (options.skip_report):
print("+ No report generation...")
else:
print("\n+ Generating a report BUSCO plot.")
outdir_report = HCGB_files.create_subfolder("report", outdir)
## get subdirs generated and call multiQC report module
givenList = []
print(
"+ Detail information for each sample could be identified in separate folders."
)
## name folder according to mode
if mode == 'genome':
BUSCO_report = HCGB_files.create_subfolder("BUSCO_assembly",
outdir_report)
elif mode == 'proteins':
BUSCO_report = HCGB_files.create_subfolder("BUSCO_annot",
outdir_report)
## generate plots
print("+ Generate summarizing plots...")
BUSCO_caller.BUSCO_plots(dataFrame_results, BUSCO_report,
options.threads)
print('\n+ Check quality plots in folder: %s' % BUSCO_report)
## TODO
## Parse BUSCO statistics in dataframe (stats_results) for discarding samples if necessary
## given a cutoff, discard or advise to discard some samples
### print statistics
stats_results.to_csv(BUSCO_report + "/BUSCO_stats.csv")
name_excel = BUSCO_report + "/BUSCO_stats.xlsx"
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
stats_results.to_excel(writer, sheet_name="BUSCO statistics")
writer.save()
print('\n+ Check quality statistics in folder: %s' % BUSCO_report)
return (dataFrame_results)
def run_ident(options):
"""
Main function acting as an entry point to the module *ident*.
Arguments:
.. seealso:: Additional information to PubMLST available datasets.
- :doc:`PubMLST datasets<../../../data/PubMLST_datasets>`
"""
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_KMA):
## information for KMA Software
species_identification_KMA.help_kma_database()
exit()
elif (options.help_MLSTar):
## information for KMA Software
MLSTar.help_MLSTar()
exit()
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
### species_identification_KMA -> most similar taxa
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Species identification")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
global Project
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "ident",
options.debug)
## let's start the process
print(
"+ Generate an species typification for each sample retrieved using:")
print("(1) Kmer alignment (KMA) software.")
print("(2) Pre-defined databases by KMA or user-defined databases.")
## get databases to check
retrieve_databases = get_options_db(options)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve_database **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(retrieve_databases)
######## KMA identification
dataFrame_kma = KMA_ident(options, pd_samples_retrieved, outdir_dict,
retrieve_databases, start_time_partial)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve results to summarize **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("dataframe_kma")
print(dataFrame_kma)
## exit if viral search
skip = False
if (len(options.kma_dbs) == 1):
for i in options.kma_dbs:
if (i == 'viral'):
print()
MLST_results = ''
options.fast = True
skip = True
## what if only plasmids?
## do edirect and MLST if bacteria
if (not skip):
dataFrame_edirect = pd.DataFrame()
######## EDirect identification
#dataFrame_edirect = edirect_ident(dataFrame_kma, outdir_dict, Debug)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve results from NCBI **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("dataFrame_edirect")
print(dataFrame_edirect)
######## MLST identification
MLST_results = MLST_ident(options, dataFrame_kma, outdir_dict,
dataFrame_edirect, retrieve_databases)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(
colored("**DEBUG: retrieve results to summarize **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("MLST_results")
print(MLST_results)
## generate summary for sample: all databases
## MLST, plasmids, genome, etc
HCGB_aes.boxymcboxface("Results Summary")
#####################################
## Summary identification results ##
#####################################
## parse results
if options.project:
final_dir = os.path.join(outdir, 'report', 'ident')
HCGB_files.create_folder(final_dir)
else:
final_dir = outdir
###
excel_folder = HCGB_files.create_subfolder("samples", final_dir)
print('+ Print summary results in folder: ', final_dir)
print('+ Print sample results in folder: ', excel_folder)
# Group dataframe results summary by sample name
sample_results_summary = dataFrame_kma.groupby(["Sample"])
## debug message
if (Debug):
print(colored("**DEBUG: sample_results_summary **", 'yellow'))
print(sample_results_summary)
##
results_summary_KMA = pd.DataFrame()
MLST_all = pd.DataFrame()
for name, grouped in sample_results_summary:
## create a excel and txt for sample
name_sample_excel = excel_folder + '/' + name + '_ident.xlsx'
name_sample_csv = outdir_dict[
name] + '/ident_summary.csv' ## check in detached mode
writer_sample = pd.ExcelWriter(
name_sample_excel, engine='xlsxwriter') ## open excel handle
## subset dataframe & print result
results_summary_toPrint_sample = grouped[[
'Sample', '#Template', 'Query_Coverage', 'Template_Coverage',
'Depth', 'Database'
]]
results_summary_toPrint_sample.to_excel(
writer_sample, sheet_name="KMA") ## write excel handle
results_summary_toPrint_sample.to_csv(
name_sample_csv) ## write csv for sample
## read MLST
if MLST_results:
if name in MLST_results:
sample_MLST = pd.read_csv(MLST_results[name],
header=0,
sep=',')
sample_MLST['genus'] = dataFrame_edirect.loc[
dataFrame_edirect['sample'] == name, 'genus'].values[0]
sample_MLST['species'] = dataFrame_edirect.loc[
dataFrame_edirect['sample'] == name, 'species'].values[0]
sample_MLST.to_excel(writer_sample,
sheet_name="MLST") ## write excel handle
## Return information to excel
MLST_all = pd.concat([MLST_all, sample_MLST])
## close excel handle
writer_sample.save()
##
name_excel = final_dir + '/identification_summary.xlsx'
print('+ Summary information in excel file: ', name_excel)
writer = pd.ExcelWriter(name_excel,
engine='xlsxwriter') ## open excel handle
## KMA dataframe: print result for sources
results_summary_KMA = dataFrame_kma[[
'Sample', '#Template', 'Query_Coverage', 'Template_Coverage', 'Depth',
'Database'
]]
## Sum plasmid and chromosome statistics ##
## sum coverage
total_coverage = results_summary_KMA.groupby(
'Sample')['Query_Coverage'].sum().reset_index()
## debug message
if (Debug):
print("*** Sum: Query_coverage ***")
print(total_coverage)
## TODO: FIX SUMMARY REPORT
results_summary_KMA = results_summary_KMA.set_index('Sample')
results_summary_KMA = results_summary_KMA.sort_values(
by=['Sample', 'Database', 'Query_Coverage'],
ascending=[True, True, True])
results_summary_KMA.to_excel(writer,
sheet_name='KMA') ## write excel handle
## write MLST
if (MLST_results):
MLST_all.to_excel(writer, sheet_name='MLST')
## write excel and close
writer.save() ## close excel handle
print("\n+ Check summary of results in file generated")
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
######################################
## update database for later usage
######################################
if not options.fast:
HCGB_aes.boxymcboxface("Update Sample Database")
## update db
print("+ Update database with samples identified")
## debug message
if (Debug):
print(colored("**DEBUG: dataFrame_edirect **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(dataFrame_edirect)
## dataFrame_edirect
file_toprint = final_dir + '/edirect_info2download.csv'
dataFrame_edirect.to_csv(file_toprint)
## update database with samples identified
data2download = dataFrame_edirect.filter(
['genus', 'species', 'strain', 'genome'])
data2download = data2download.rename(columns={
'genome': 'NCBI_assembly_ID',
'strain': 'name'
})
NCBI_folder = os.path.abspath(options.database) + '/NCBI'
database_generator.NCBI_DB(data2download, NCBI_folder, Debug)
else:
print(
"+ No update of the database has been requested using option --fast"
)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting identification module.")
return ()
def edirect_ident(dataFrame, outdir_dict, Debug):
"""Connect to NCBI for information retrieval
This functions uses the software edirect_ to connect to NCBI and retrieve some information regarding samples, assemblies, publications, etc.
:param dataFrame: pandas dataframe for samples to process. Result from :func:`BacterialTyper.modules.ident.KMA_ident`.
:param outdir_dict: dictionary containing information for each sample of the output folder for this process.
:type dataFrame: pandas.DataFrame()
:type outdir_dict: Dictionary
:return: Information of the identification
:rtype: pandas.DataFrame()
See example of returned dataframe in file :file:`/devel/results/edirect_download_results.csv` here:
.. include:: ../../devel/results/edirect_download_results.csv
:literal:
.. seealso:: This function depends on other ``BacterialTyper`` functions called:
- :func:`BacterialTyper.scripts.functions.get_info_file`
- :func:`BacterialTyper.scripts.functions.read_time_stamp`
- :func:`BacterialTyper.scripts.functions.print_time_stamp`
- :func:`BacterialTyper.scripts.functions.optimize_threads`
- :func:`BacterialTyper.scripts.functions.create_subfolder`
- :func:`BacterialTyper.scripts.functions.boxymcboxface`
- :func:`BacterialTyper.scripts.functions.is_non_zero_file`
- :func:`BacterialTyper.scripts.edirect_caller.generate_docsum_call`
- :func:`BacterialTyper.scripts.edirect_caller.generate_xtract_call`
.. include:: ../../links.inc
"""
################################################
## TODO: What to do if multi-isolate sample?
################################################
## edirect
HCGB_aes.boxymcboxface("EDirect information")
print("+ Connect to NCBI to get information from samples identified...")
## create dataframe to return results
edirect_frame = pd.DataFrame(columns=("sample", "genus", "species",
"strain", "BioSample", "genome",
"Plasmids"))
## debugging messages
if Debug:
print("*******************************************************")
print("Dataframe sample_results: ")
# Group dataframe sample name
sample_results = dataFrame.groupby(["Sample"])
for name, grouped in sample_results:
## debugging messages
if Debug:
print("Name: ", name)
print(grouped)
## use edirect to get Species_name and entry for later identification
edirect_folder = HCGB_files.create_subfolder('edirect',
outdir_dict[name])
## chromosome match
if (len(grouped.loc[grouped['Database'] == 'bacteria.ATG']
['#Template']) == 0):
if Debug:
print("Name: ", name)
print("No chromosome match identified by kmer")
genus = ''
species = ''
BioSample_name = ''
AssemblyAcc = ''
else:
nucc_entry = grouped.loc[grouped['Database'] == 'bacteria.ATG'][
'#Template'].values[0].split()
## e.g. NZ_CP029680.1 Staphylococcus aureus strain AR_0215 chromosome, complete genome
##
out_docsum_file = edirect_folder + '/nuccore_docsum.txt'
tmp_species_outfile = edirect_folder + '/info.csv'
filename_stamp = edirect_folder + '/.success_species'
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print(
colored(
"\tA previous command generated results on: %s [%s]" %
(stamp, name), 'yellow'))
status = True
else:
edirect_caller.generate_docsum_call('nuccore', nucc_entry[0],
out_docsum_file)
status = edirect_caller.generate_xtract_call(
out_docsum_file, 'DocumentSummary',
'Organism,BioSample,AssemblyAcc,Strain',
tmp_species_outfile)
########################################
## get information from edirect call
########################################
if not status:
print("NO INFORMATION")
continue
taxa_name_tmp = HCGB_main.get_info_file(tmp_species_outfile)
Organism = taxa_name_tmp[0].split(',')[0].split()
genus = Organism[0] ## genus
species = Organism[1] ## species
BioSample_name = taxa_name_tmp[0].split(',')[1] ## BioSample
AssemblyAcc = taxa_name_tmp[0].split(',')[2] ## AssemblyAcc
## sometimes strain is missing
if len(taxa_name_tmp[0].split(',')) > 3:
strain = taxa_name_tmp[0].split(',')[3] ## strain
else:
strain = 'NaN'
## get GenBank accession ID
out_docsum_file_assembly = edirect_folder + '/assembly_docsum.txt'
AssemblyAcc_outfile = edirect_folder + '/AssemblyAcc.csv'
edirect_caller.generate_docsum_call('assembly', AssemblyAcc,
out_docsum_file_assembly)
edirect_caller.generate_xtract_call(out_docsum_file_assembly,
'DocumentSummary', 'Genbank',
AssemblyAcc_outfile)
## some error occurred
if not HCGB_main.is_non_zero_file(out_docsum_file_assembly):
continue
## Is it better to download Refseq or Genbank?
## https://www.quora.com/What-is-the-difference-between-Refseq-and-Genbank
GenbankAcc = HCGB_main.get_info_file(AssemblyAcc_outfile)
if Debug:
print("Sample: ", name)
print("Genbank Acc: ", GenbankAcc[0])
## plasmid match
group_plasmid = grouped.loc[grouped['Database'] == 'plasmids.T']
plasmid_entries = group_plasmid['#Template'].tolist()
## e.g. NZ_CP029083.1 Staphylococcus aureus strain AR464 plasmid unnamed1, complete sequence
plasmid_entries_str = ",".join([i.split()[0] for i in plasmid_entries])
## save edirect_frame
#("sample", "taxa", strain, genome "BioSample", "Plasmids"))
edirect_frame.loc[len(edirect_frame)] = (name, genus, species, strain,
BioSample_name, GenbankAcc[0],
plasmid_entries_str)
stamp = HCGB_time.print_time_stamp(filename_stamp)
## debugging messages
if Debug:
print("*******************************************************")
return (edirect_frame)
def run_report(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_spaTyper):
## help_format option
get_spa_typing.help_spaTyper()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
## set default
options.batch = False
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Report generation module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## call assemble using spades
start_time_partial = start_time_total
## absolute path for in & out
options.database = os.path.abspath(options.database)
global input_dir
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
global Project
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
##
print("\n+ Get project information:")
## get files: trimm, assembly, annotation
pd_samples_retrieved = database_user.get_userData_files(options, input_dir)
pd_samples_retrieved['new_name'] = pd_samples_retrieved['name']
## get info: profile, ident, cluster, MGE
pd_samples_info = database_user.get_userData_info(options, input_dir)
## get databases to list
#retrieve_databases = get_options_db(options)
## create output files
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "report",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
print(colored("**DEBUG: pd_samples_info **", 'yellow'))
print(pd_samples_info)
## generate output folder, if necessary
print(
"\n\n\n+ Generate a report summarizing analysis and sample information"
)
if not options.project:
HCGB_files.create_folder(outdir)
outdir_report = outdir
else:
### report generation
outdir_report = HCGB_files.create_subfolder("report", outdir)
## create report with all data
summary_report = HCGB_files.create_subfolder("summary_report",
outdir_report)
print("Folder: ", summary_report)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
########################################
## create species specific report if any
########################################
if (options.species_report):
## Saureus
if options.species_report == "Saureus":
Saureus_specific(pd_samples_retrieved, pd_samples_info, options,
summary_report, outdir_dict)
## else
## to add accordingly
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###########################################################
## create gene fasta sequences retrieval if desired
###########################################################
if options.genes_ids_fasta:
## given a list of genes ids, retrieve sequence for all samples from profile
if os.path.isfile(os.path.abspath(options.genes_ids_fasta)):
in_file = os.path.abspath(options.genes_ids_fasta)
gene_names = [line.rstrip('\n') for line in open(in_file)]
print(
'+ Retrieve selected genes sequences from the profile analysis for each sample.'
)
print('+ Searching gene:')
## get profiles available
results_geneIDs = pd.DataFrame(columns=('sample', 'gene', 'id',
'sequence'))
sample_frame = pd_samples_info.groupby(["name"])
for g in gene_names:
print("\t+", g)
for name, cluster_df in sample_frame:
my_list_profiles = cluster_df.loc[
cluster_df['tag'] == 'profile']['ext'].to_list()
if options.debug:
print("name: ", name)
print("my_list_profiles:")
print(my_list_profiles)
for p in my_list_profiles:
main_profile_folder = cluster_df.loc[
cluster_df['ext'] == p]['dirname'].to_list()[0]
p = p.lower()
if p == 'vfdb':
p = p + '_full'
profile_folder = os.path.join(main_profile_folder, p)
(seq_id, seq_sequence
) = retrieve_genes.retrieve_genes_ids_sequences(
profile_folder, g, Debug)
if (seq_id):
## save results
results_geneIDs.loc[len(results_geneIDs)] = (
name, g, seq_id, seq_sequence)
## save for each gene in a separate fasta file
list_of_genes = set(results_geneIDs['gene'].to_list())
## debug
if Debug:
print("** DEBUG **")
print(results_geneIDs)
print(list_of_genes)
## Save results
genes_folder = HCGB_files.create_subfolder('genes', summary_report)
for gene_retrieved in list_of_genes:
this_frame = results_geneIDs[results_geneIDs['gene'] ==
gene_retrieved]
gene_retrieved_file = os.path.join(genes_folder, gene_retrieved)
gene_retrieved_fasta = gene_retrieved_file + ".fasta"
gene_retrieved_info = gene_retrieved_file + "_info.txt"
fasta_hd = open(gene_retrieved_fasta, 'w')
info_hd = open(gene_retrieved_info, 'w')
for item, row in this_frame.iterrows():
string2write = ">" + row['sample'] + '_' + row[
'gene'] + '\n' + row['sequence'] + '\n'
string2write_info = row['sample'] + '\t' + row[
'gene'] + '\t' + row['id'] + '\n'
fasta_hd.write(string2write)
info_hd.write(string2write_info)
fasta_hd.close()
info_hd.close()
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
########################################
## create gene promoter fasta sequences retrieval if desired
########################################
if options.promoter_bp:
## retrieve as many bp as necessary from genes_ids_fasta
print("** THIS OPTION IS NOT IMPLEMENTED YET... **")
#get_promoter.get_promoter(file, geneOfInterest, basePairs, sampleName, option, debug=False):
########################################
## create gene specific report if any
########################################
if options.genes_ids_profile:
if options.species_report == "Saureus":
if Debug:
print("** options.genes_ids_profile **")
print("Analysis already done for Saureus")
else:
in_file = os.path.abspath(options.genes_ids_profile)
gene_names = [line.rstrip('\n') for line in open(in_file)]
results_Profiles = retrieve_genes.get_genes_profile(
pd_samples_info, gene_names, options.debug, "name")
if options.debug:
print("results_Profiles")
print(results_Profiles)
## open excel writer
name_excel = summary_report + '/gene_ids_profile.xlsx'
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
results_Profiles.to_excel(writer, sheet_name="gene_ids")
## close
writer.save()
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###############################################
## Search for any additional fasta sequence
###############################################
if options.genes_fasta:
## given a list of fasta sequences search using blast against proteins annotated or genome
print("** THIS OPTION IS NOT IMPLEMENTED YET... **")
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Report generation module.")
return ()
def run_phylo(options):
"""
Main function acting as an entry point to the module *phylo*.
"""
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Phylogenetic reconstruction")
print ("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir=""
## set mode: project/detached
## Project mode as default
project_mode=True
if (options.detached):
options.project = False
project_mode=False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get the database
options.database = os.path.abspath(options.database)
### parse the reference
print ("+ Retrieve the reference...")
reference_gbk_file = get_reference_gbk(options)
## generate output folder, if necessary
print ("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
##################################
## select samples and map
####################################
print ("+ Retrieve samples to map available...")
dict_folders = map_samples(options, reference_gbk_file, input_dir, outdir)
if Debug:
print (colored("**DEBUG: dict_folders **", 'yellow'))
print (dict_folders)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
##################################
## Create core alingment
##################################
outdir_report = HCGB_files.create_subfolder("report", outdir)
phylo_dir = HCGB_files.create_subfolder("phylo", outdir_report)
analysis_dir = HCGB_files.create_subfolder(options.name, phylo_dir)
snippy_dir = HCGB_files.create_subfolder("snippy", analysis_dir)
list_folders = list(dict_folders.values())
options_string = ""
variant_calling.snippy_core_call(list_folders, options_string, options.name,
snippy_dir, options.output_format, Debug)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## snp distance matrix
snp_distance_dir = HCGB_files.create_subfolder("snp_distance", analysis_dir)
name_matrix = os.path.join(snp_distance_dir, "snp_matrix_" + options.name)
countGaps = False
aln_file = os.path.join(snippy_dir, options.name + '.aln')
phylo_parser.get_snp_distance(aln_file, options.output_format, countGaps, name_matrix, Debug)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## phylogenetic analysis
iqtree_output = HCGB_files.create_subfolder("iqtree", analysis_dir)
phylo_parser.ml_tree(snippy_dir, options.name, options.threads, iqtree_output, Debug)
## time stamp
start_time_partial = HCGB_files.timestamp(start_time_total)
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting Annotation module.")
return()
def help_fastq_format():
"""
Explanation of fastq format details.
"""
HCGB_aes.boxymcboxface("Name format for samples")
print("Format for fastq files can be:")
print(
"name.fastq.gz, name_1.fastq.gz, name_R2.fastq.gz, name_L001_R1.fastq.gz, name_L001_R1_001.fastq.gz etc."
)
print(
"\nThere are many options and here we provide some guidelines on the name format."
)
print("\n")
HCGB_aes.print_sepLine("*", 20, "red")
print("[1] Length limitation")
HCGB_aes.print_sepLine("*", 20, "red")
print("There is a limitation for the sample ID ('name') of 25 characters.")
print(
colored(
"** BacterialTyper provides an option to rename samples if necessary: module prep option --rename **",
'yellow'))
print("\n")
HCGB_aes.print_sepLine("*", 20, "red")
print("[2] Single end files")
HCGB_aes.print_sepLine("*", 20, "red")
print(
colored(
'** Use option --single-end in the different BacterialTyper modules **',
'yellow'))
print("name.fastq.gz")
print("name.fastq")
print("name.fq")
print("\n")
HCGB_aes.print_sepLine("*", 20, "red")
print("[3] Paired-end files")
HCGB_aes.print_sepLine("*", 20, "red")
print(
"Paired-end files are full supported. The format for these files are:")
print("Read1 => name_1.fastq.g or name_R1.fastq.gz")
print("Read2 => name_2.fastq.gz or name_R2.fastq.gz")
print(
colored('** See additional details for Lane information **', 'yellow'))
print("\n")
HCGB_aes.print_sepLine("*", 55, "red")
print("[4] Lane information:")
HCGB_aes.print_sepLine("*", 55, "red")
print(
"In some cases, files might contain lane information (*L00x* and/or *00x*)."
)
print(
"BacterialTyper supports these names as long as follow these examples:"
)
print("name_L00x_R1.fastq.gz\tname_L00x_R2.fastq.gz")
print("name_L00x_1.fastq.gz\tname_L00x_2.fastq.gz")
print("name_L00x_R1.fastq.gz\tname_L00x_R2.fastq.gz")
print("name_L00x_R1_00x.fastq.gz\tname_L00x_R2_00x.fastq.gz")
print("\n")
print(
"Sometimes it might be appropriate to include lane tags (*L00X*) within the name."
)
print(colored("** Use option --include-lane within each module", 'yellow'))
print(
colored(
"\n** If you need to merge fastq files from different lanes, use option within module prep **",
'yellow'))
print("As an example:")
print(colored("\n** Option --merge within module prep **", 'yellow'))
print("sample1_L001_R1.fastq.gz\tsample1_L001_R2.fastq.gz")
print("sample1_L002_R1.fastq.gz\tsample1_L002_R2.fastq.gz")
print("sample1_L003_R1.fastq.gz\tsample1_L003_R2.fastq.gz")
print("sample1_L004_R1.fastq.gz\tsample1_L004_R2.fastq.gz")
print("Result:")
print("--------------------------------------------------")
print("sample1_R1.fastq.gz\tsample1_R2.fastq.gz")
print("\n")
print(
colored("\n** Option --merge-by-lane within module prep **", 'yellow'))
print("sample1_L001_R1_001.fastq.gz\tsample1_L001_R2_001.fastq.gz")
print("sample1_L001_R1_002.fastq.gz\tsample1_L001_R2_002.fastq.gz")
print("sample1_L002_R1_001.fastq.gz\tsample1_L002_R2_001.fastq.gz")
print("sample1_L002_R1_002.fastq.gz\tsample1_L002_R2_002.fastq.gz")
print("--------------------------------------------------")
print("Result:")
print("sample1_L001_R1.fastq.gz\tsample1_L001_R2.fastq.gz")
print("sample1_L002_R1.fastq.gz\tsample1_L002_R2.fastq.gz")
print(
colored("** Remember to use option --include_lane within each module",
'yellow'))
print("\n")
HCGB_aes.print_sepLine("*", 55, "red")
print("[5] Include all information:")
HCGB_aes.print_sepLine("*", 55, "red")
print(
"In some cases, files might contain other information and it is necessay to "
+ "include it all as a tag nane. See as an example:")
print("sample1_L001_XYZ_R1_001.fastq.gz\tsample1_L001_XYZ_R2_001.fastq.gz")
print(
colored("** Remember to use option --include_all within each module",
'yellow'))
print(
colored(
"** It might be appropiate to change samples names using --rename option under prep module",
'yellow'))
print("\n")
HCGB_aes.print_sepLine("*", 15, "red")
print("[6] Extensions:")
HCGB_aes.print_sepLine("*", 15, "red")
print(
"name_L00x_R2.fastq\tname_L00x_R2.fq\nname_L00x_R2.fastq.gz\tname_L00x_R2.fq.gz"
)
print("\n")
def run_assembly(options):
"""Main function of the assemble module.
It assembles each sample using SPADES_ and checks quality using BUSCO_ software and database.
.. seealso:: This function depends on other BacterialTyper and HCGB functions called:
- :func:`BacterialTyper.scripts.BUSCO_caller.print_help_BUSCO`
- :func:`BacterialTyper.scripts.multiQC_report.multiqc_help`
- :func:`BacterialTyper.modules.qc.BUSCO_check`
- :func:`HCGB.sampleParser`
- :func:`HCGB.functions.aesthetics_functions`
- :func:`HCGB.functions.time_functions`
- :func:`HCGB.functions.main_functions`
- :func:`HCGB.functions.file_functions`
.. include:: ../../links.inc
"""
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
exit()
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Assembly module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "assemble",
options.debug)
### call assemble using spades
start_time_partial = start_time_total
start_time_partial_assembly = start_time_partial
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
HCGB_aes.debug_message("options.threads: " + str(options.threads),
"yellow")
HCGB_aes.debug_message("max_workers: " + str(max_workers_int),
"yellow")
HCGB_aes.debug_message("cpu_here: " + str(threads_job), "yellow")
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
# We can use a with statement to ensure threads are cleaned up promptly
print('+ Running modules SPADES...')
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
## send for each sample
commandsSent = {
executor.submit(check_sample_assembly, name, outdir_dict[name],
sorted(cluster["sample"].tolist()), threads_job):
name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
## functions.timestamp
print("\n+ Assembly of all samples finished: ")
start_time_partial = HCGB_time.timestamp(start_time_partial_assembly)
##
if (assembly_stats):
###################
if Debug:
HCGB_aes.debug_message("assembly_stats dictionary", "yellow")
print(assembly_stats)
## create single file
get_assembly_stats_all(assembly_stats, outdir, Debug)
### symbolic links
print("+ Retrieve all genomes assembled...")
### BUSCO check assembly
if (options.no_BUSCO):
print()
else:
results = qc.BUSCO_check(outdir, outdir, options, start_time_partial,
"genome")
## print to file results
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Assembly module.")
return ()
def run_database(options):
## init time
start_time_total = time.time()
start_time_partial = start_time_total
## debugging messages
global Debug
if (options.debug):
Debug = True
print("[Debug mode: ON]")
else:
Debug = False
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Database")
print("--------- Starting Process ---------")
HCGB_time.print_time()
kma_bin = set_config.get_exe("kma")
######################################################
## print further information if requested
if (options.help_ARIBA):
print("ARIBA databases information:")
ariba_caller.help_ARIBA()
exit()
elif (options.help_BUSCO):
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_KMA):
species_identification_KMA.help_kma_database()
exit()
######################################################
## create folder
## absolute
options.path = os.path.abspath(options.path)
HCGB_files.create_folder(options.path)
#########
if Debug:
print(colored("DEBUG: absolute path folder: " + options.path,
'yellow'))
##########
## NCBI ##
##########
## if any NCBI options provided
if any([options.ID_file, options.descendant]):
## create folders
NCBI_folder = HCGB_files.create_subfolder('NCBI', options.path)
if (options.ID_file):
## get path and check if it is file
abs_path_file = os.path.abspath(options.ID_file)
if os.path.isfile(abs_path_file):
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check NCBI ids provided ---------\n")
HCGB_aes.print_sepLine("*", 70, False)
## get file information
print("\t+ Obtaining information from file: %s" %
abs_path_file)
strains2get = HCGB_main.get_data(abs_path_file, ',', '')
dataBase_NCBI = database_generator.NCBI_DB(
strains2get, NCBI_folder, Debug)
#########
if Debug:
print(colored("DEBUG: NCBI data provided: ", 'yellow'))
print(options.ID_file)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## strains downloaded would be included to a kma index
## Get all entries belonging to this taxon provided
if (options.descendant):
#########
if Debug:
print(colored("DEBUG: NCBI descendant option: ON ", 'yellow'))
print()
HCGB_aes.print_sepLine("*", 70, False)
print(
"--------- Check descendant NCBI taxonomy ids provided ---------\n"
)
HCGB_aes.print_sepLine("*", 70, False)
## [TODO]
dataBase_NCBI = database_generator.NCBI_descendant(
options.descendant, NCBI_folder, Debug)
##############################################################
## update KMA database with NCBI information retrieved
##############################################################
print('\n\n+ Update database for later identification analysis...')
list_of_files = dataBase_NCBI['genome'].tolist()
kma_db = HCGB_files.create_subfolder('KMA_db', options.path)
genbank_kma_db = HCGB_files.create_subfolder('genbank', kma_db)
print('+ Database to update: ', genbank_kma_db)
species_identification_KMA.generate_db(list_of_files, 'genbank_KMA',
genbank_kma_db, 'new', 'batch',
Debug, kma_bin)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
###############
## user_data ##
###############
if options.project_folder:
##
dataBase_user = pd.DataFrame()
## get absolute path
abs_project_folder = os.path.abspath(options.project_folder)
if os.path.exists(abs_project_folder):
#########
if Debug:
print(
colored("DEBUG: User provides folder containing project",
'yellow'))
print()
HCGB_aes.print_sepLine("*", 70, False)
print("--------- Check user provided project folder ---------")
HCGB_aes.print_sepLine("*", 70, False)
dataBase_user = database_user.update_database_user_data(
options.path, abs_project_folder, Debug, options)
else:
print(
colored(
"ERROR: Folder provided does not exists: %s" %
options.project_folder, 'red'))
exit()
##############################################################
## update KMA database with user_data information retrieved
##############################################################
print('\n\n+ Update database for later identification analysis...')
list_of_files = dataBase_user['genome'].tolist()
kma_db = HCGB_files.create_subfolder('KMA_db', options.path)
user_kma_db = HCGB_files.create_subfolder('user_data', kma_db)
print('+ Database to update: ', user_kma_db)
species_identification_KMA.generate_db(list_of_files, 'userData_KMA',
user_kma_db, 'new', 'batch',
Debug, kma_bin)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
##########
## ARIBA
##########
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check ARIBA parameters provided --------")
HCGB_aes.print_sepLine("*", 50, False)
if (options.no_ARIBA):
print("+ No ARIBA databases would be downloaded...")
#########
if Debug:
print(colored("DEBUG: No option ARIBA", 'yellow'))
else:
#functions.print_sepLine("*",50, False)
### ariba list databases
ariba_dbs_list = ['CARD', 'VFDB']
if (options.no_def_ARIBA):
ariba_dbs_list = options.ariba_dbs
else:
if (options.ariba_dbs):
ariba_dbs_list = ariba_dbs_list + options.ariba_dbs
ariba_dbs_list = set(ariba_dbs_list)
#########
if Debug:
print(colored("DEBUG: Option ARIBA", 'yellow'))
print(options.ariba_dbs)
ariba_caller.download_ariba_databases(ariba_dbs_list, options.path,
Debug, options.threads)
### ariba list databases
if (options.ariba_users_fasta):
print(
"+ Generate ARIBA database for databases provided: prepare fasta and metadata information"
)
#########
if Debug:
print(colored("DEBUG: Option user ARIBA db", 'yellow'))
print(ariba_users_fasta)
print(ariba_users_meta)
## [TODO]:
## ariba prepareref fasta and metadata
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
#########
## kma ##
#########
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check KMA parameters provided ----------")
kma_database = options.path + '/KMA_db'
HCGB_files.create_folder(kma_database)
## types: bacteria, archaea, protozoa, fungi, plasmids, typestrains
## downloads all "bacterial" genomes from KMA website
## kma: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/
print(
"+ Retrieving information from: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder website"
)
## KMA databases to use
## only user dbs
if (options.no_def_kma):
if (options.kma_dbs):
print("+ Only user databases selected will be indexed...")
else:
print("+ No databases selected.")
print(colored("ERROR: Please select a kma database.", 'red'))
exit()
## default dbs + user
else:
kma_dbs = ["bacteria", "plasmids"]
## default dbs + user
if (options.kma_dbs):
options.kma_dbs = options.kma_dbs + kma_dbs
options.kma_dbs = set(options.kma_dbs)
else:
options.kma_dbs = kma_dbs
#########
if Debug:
print(colored("DEBUG: options.kma_dbs", 'yellow'))
print(options.kma_dbs)
## Get databases
for db in options.kma_dbs:
print(colored("\n+ " + db, 'yellow'))
db_folder = HCGB_files.create_subfolder(db, kma_database)
species_identification_KMA.download_kma_database(db_folder, db, Debug)
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###########
## BUSCO ##
###########
if (options.BUSCO_dbs):
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check BUSCO datasets provided ---------")
BUSCO_folder = HCGB_files.create_subfolder("BUSCO", options.path)
#########
if Debug:
print(colored("DEBUG: options.BUSCO_dbs", 'yellow'))
print(options.BUSCO_dbs)
print("+ BUSCO datasets would be downloaded when executed...")
#BUSCO_caller.BUSCO_retrieve_sets(options.BUSCO_dbs, BUSCO_folder)
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
print("\n*************** Finish *******************\n")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Database module.\n")
return ()
def ARIBA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases,
start_time_partial):
HCGB_aes.boxymcboxface("ARIBA Identification")
##################
## check status ##
##################
databases2use = [] ## path, db name
card_trick_info = ""
print('+ Check databases status: ')
for index, db2use in retrieve_databases.iterrows():
## index_name
if (db2use['source'] == 'ARIBA'):
index_status = ariba_caller.check_db_indexed(db2use['path'], 'YES')
if (index_status == True):
#print (colored("\t+ Databases %s seems to be fine...\n\n" % db2use['db'], 'green'))
databases2use.append([db2use['path'], db2use['db']])
## prepare card database ontology for later
if (db2use['db'] == 'card'):
card_trick_info = card_trick_caller.prepare_card_data(
options.database)
## check status of other databases if any
# else:
## debug message
if (Debug):
print(colored("**DEBUG: databases2use\n**", 'yellow'))
print(databases2use)
if (card_trick_info):
print(
colored("**DEBUG: card_trick_info: " + card_trick_info + " **",
'yellow'))
######################################################
## Start identification of samples
######################################################
print("\n+ Send ARIBA identification jobs...")
## get outdir folders
outdir_samples = pd.DataFrame(columns=('sample', 'dirname', 'db',
'output'))
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
for name, cluster in sample_frame:
for db2use in databases2use:
tmp = get_outfile(outdir_dict[name], name, db2use[0])
outdir_samples.loc[len(outdir_samples)] = (name, outdir_dict[name],
db2use[1], tmp)
## multi-index
outdir_samples = outdir_samples.set_index(['sample', 'db'])
## debug message
if (Debug):
print(colored("**DEBUG: outdir_samples **", 'yellow'))
print(outdir_samples)
######################################################
## send for each sample
######################################################
## ariba assembly cutoff
if not (options.ARIBA_cutoff):
options.ARIBA_cutoff = 0.90
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
## loop
results_df = pd.DataFrame()
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
for db2use in databases2use:
print(colored("+ Working with database: " + db2use[1], 'yellow'))
## send for each sample
commandsSent = {
executor.submit(
ariba_run_caller,
db2use[0],
db2use[1], ## database path & dbname
sorted(cluster["sample"].tolist()), ## files
outdir_samples.loc[(name, db2use[1]), 'output'], ## output
threads_job,
options.ARIBA_cutoff): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
print("+ Jobs finished for database %s ..." % db2use[1])
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
print()
print(
"+ Collecting information for each sample analyzed for database: "
+ db2use[1])
## check results for each database
results_df_tmp = virulence_resistance.check_results(
db2use[1], outdir_samples, options.ARIBA_cutoff,
card_trick_info)
results_df = pd.concat([results_df, results_df_tmp])
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
######################################################
## Generate final report for all samples
######################################################
## ariba summary results all samples
print(
"\n + Generate a summary file for all samples and one for each database employed..."
)
## parse results
if Project:
final_dir = input_dir + '/report/profile'
HCGB_files.create_folder(final_dir)
else:
final_dir = os.path.abspath(options.output_folder)
##
vfdb = False
subfolder = HCGB_files.create_subfolder("ariba_summary", final_dir)
## subfolder_samples = functions.create_subfolder("samples", final_dir) ## TODO: Copy all xlsx files to a common folder. Is it necessary?
## open excel writer
name_excel = final_dir + '/profile_summary.xlsx'
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
for database, data in outdir_samples.groupby(level='db'): ## fix
report_files_databases = {}
for sample, data2 in data.groupby(level='sample'): ## fix
file_report = data2.loc[sample, database]['output'] + '/report.tsv'
if os.path.isfile(file_report): ## check if exists
report_files_databases[sample] = file_report
outfile_summary = subfolder + "/"
if database.endswith('card_prepareref/'):
outfile_summary = outfile_summary + 'CARD_summary'
name_db = 'CARD'
elif database.endswith('vfdb_full_prepareref/'):
outfile_summary = outfile_summary + 'VFDB_summary'
name_db = 'VFDB'
vfdb = True
else:
## TODO: check if there are multiple 'other' databases
## Different databases provided (different to VFDB and CARD) would collapse file
outfile_summary = outfile_summary + 'Other_summary'
name_db = 'other'
## call ariba summary to summarize results
csv_all = ariba_caller.ariba_summary_all(outfile_summary,
report_files_databases)
if not csv_all == 'NaN':
csv2excel = pd.read_csv(csv_all, header=0, sep=',')
## write excel
name_tab = name_db + '_found'
csv2excel.to_excel(writer, sheet_name=name_tab)
## results_df contains excel and csv files for each sample and for each database
list_databases = set(results_df['database'].to_list())
for db in list_databases:
df_db = results_df[results_df['database'] == db]['csv']
dict_samples = df_db.to_dict()
merge_df = pd.DataFrame()
for sample in dict_samples:
if os.path.isfile(dict_samples[sample]):
df = pd.read_csv(dict_samples[sample], header=0, sep=",")
df = df.set_index('Genes')
df2 = df.rename(columns={'Status': sample}, inplace=True)
df2 = df[[sample]]
## add to a common dataframe
merge_df = pd.concat([merge_df, df2], axis=1, sort=True)
merge_df.fillna("NaN", inplace=True)
trans_df = merge_df.transpose()
## write excel
name_tab = db + '_all'
trans_df.to_excel(writer, sheet_name=name_tab)
## close
writer.save()
######################################################
## print additional information for VFDB
######################################################
if (vfdb):
print("\n\n")
HCGB_aes.print_sepLine("*", 50, False)
print("+ Check VFDB details in files downloaded from vfdb website:")
files_VFDB = virulence_resistance.check_VFDB(final_dir +
'/VFDB_information')
HCGB_aes.print_sepLine("*", 50, False)
######################################################
print("\n+ Please check additional summary files generated at folder ",
final_dir)
print("+ Go to website: https://jameshadfield.github.io/phandango/#/")
print(
"+ For each database upload files *phandango.csv and *phandango.tre and visualize results"
)
def run_BacterialTyper(options):
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
elif (options.help_Prokka):
## information for Prokka
annotation.print_list_prokka()
exit()
elif (options.help_Mash):
## information for Min Hash Software
min_hash_caller.helpMash()
exit()
elif (options.help_ARIBA):
## information for ARIBA
ariba_caller.help_ARIBA()
exit()
elif (options.help_trimm_adapters):
## help on trimm adapters
trimmomatic_call.print_help_adapters()
exit()
elif (options.help_KMA):
## information for KMA Software
species_identification_KMA.help_kma_database()
exit()
elif (options.help_MLSTar):
## information for KMA Software
MLSTar.help_MLSTar()
exit()
elif (options.help_PhiSpy):
## information for PhiSpy software
bacteriophage.help_PhiSpy()
exit()
elif (options.help_MGE_analysis):
## information for MGE module analysis
MGE.help_MGE_analysis()
exit()
elif (options.help_input_MGE):
## information for PhiSpy
MGE.help_input_MGE()
exit()
###
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("BacterialTyper analysis")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
if (options.project):
outdir = input_dir
elif (options.detached):
outdir = os.path.abspath(options.output_folder)
def run_cluster(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_Mash):
## information for Min Hash Software
min_hash_caller.helpMash()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Clustering samples")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
if options.reads:
if options.noTrim:
## raw reads
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq",
("fastq", "fq", "fastq.gz", "fq.gz"), options.debug)
else:
## trimm reads
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## keep only R1 reads if paired-end
if options.pair:
pd_samples_retrieved = pd_samples_retrieved.loc[
pd_samples_retrieved['read_pair'] == "R1"]
else:
## default
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
# exit if empty
if pd_samples_retrieved.empty:
print(
"No data has been retrieved from the project folder provided. Exiting now..."
)
exit()
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "mash",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: outdir_dict **", 'yellow'))
print(outdir_dict)
## get databases to check
retrieve_databases = get_options_db(options)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## remove samples if specified
if options.ex_sample:
ex_samples = HCGB_main.get_info_file(options.ex_sample)
retrieve_databases = retrieve_databases.loc[~retrieve_databases.index.
isin(ex_samples)]
## debug message
if (Debug):
print(colored("**DEBUG: retrieve_database **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(retrieve_databases)
## check if all samples in user_data or genbank are indexed
siglist_all = []
for index, row in retrieve_databases.iterrows():
if not row['path'] == 'NaN':
if (Debug):
HCGB_aes.print_sepLine("*", 25, False)
print(row)
if all([
int(options.kmer_size) == int(row['ksize']),
int(options.n_sketch) == int(row['num_sketch'])
]):
siglist_all.append(
min_hash_caller.read_signature(row['path'],
options.kmer_size))
continue
## index assembly or reads...
(sigfile, siglist) = generate_sketch(row['folder'], row['original'],
index, options.kmer_size,
options.n_sketch, Debug)
retrieve_databases.loc[index]['path'] = sigfile
retrieve_databases.loc[index]['ksize'] = options.kmer_size
retrieve_databases.loc[index]['num_sketch'] = options.n_sketch
siglist_all.append(siglist)
### Cluster project samples
print(colored("\n+ Collect project data", 'green'))
print("+ Generate mash sketches for each sample analyzed...")
pd_samples_retrieved = pd_samples_retrieved.set_index('name')
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieved **", 'yellow'))
print(pd_samples_retrieved)
## init dataframe for project data
colname = ["source", "name", "path", "original", "ksize", "num_sketch"]
pd_samples_sketched = pd.DataFrame(columns=colname)
for index, row in pd_samples_retrieved.iterrows():
if index in retrieve_databases.index:
print(
colored(
'\t+ Sketched signature (%s) available within user data...'
% index, 'yellow'))
continue
this_sig = outdir_dict[index] + '/' + index + '.sig'
if os.path.exists(this_sig):
## File signature might exist
## read original
file2print = outdir_dict[index] + '/.original'
if not os.path.exists(file2print):
original = ['NaN']
else:
original = HCGB_main.readList_fromFile(file2print)
if all([
int(options.kmer_size) == int(original[1]),
int(options.n_sketch) == int(original[2])
]):
siglist_all.append(
min_hash_caller.read_signature(this_sig,
options.kmer_size))
pd_samples_sketched.loc[len(pd_samples_sketched)] = (
'project_data', index, this_sig, row['sample'],
options.kmer_size, options.n_sketch)
print(
colored(
'\t+ Sketched signature available (%s) in project folder...'
% index, 'green'))
continue
print(
colored('\t+ Sketched signature to be generated: (%s)...' % index,
'yellow'))
## index assembly or reads...
(sigfile, siglist) = generate_sketch(outdir_dict[index], row['sample'],
index, options.kmer_size,
options.n_sketch, Debug)
pd_samples_sketched.loc[len(pd_samples_sketched)] = ('project_data',
index, sigfile,
row['sample'],
options.kmer_size,
options.n_sketch)
siglist_all.append(siglist)
print("\n+ Clustering sequences...")
pd_samples_sketched = pd_samples_sketched.set_index('name')
####
if retrieve_databases.empty:
cluster_df = pd_samples_sketched
else:
tmp = retrieve_databases[[
'source', 'db', 'path', 'original', 'ksize', 'num_sketch'
]]
tmp = tmp.rename(columns={'db': 'name'})
tmp.set_index('name')
if (Debug):
print(colored("**DEBUG: tmp **", 'yellow'))
print(tmp)
## merge both dataframes
cluster_df = pd.concat([pd_samples_sketched, tmp],
join='inner',
sort=True)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_sketched **", 'yellow'))
print(pd_samples_sketched)
print(colored("**DEBUG: cluster_df **", 'yellow'))
print(cluster_df)
print(colored("**DEBUG: Signatures **", 'yellow'))
print(siglist_all)
print(colored("**DEBUG: length siglist_all **", 'yellow'))
print(len(siglist_all))
## Assign Colors colorLabels
color_df = cluster_df.filter(["source"], axis=1)
color_df["color"] = "r" ## red::genbank
## project data
project_data = list(color_df[color_df["source"] == "project_data"].index)
color_df.loc[color_df.index.isin(project_data),
"color"] = "g" ## green::project_data
## user_data
user_data = list(color_df[color_df["source"] == "user_data"].index)
color_df.loc[color_df.index.isin(user_data),
"color"] = "b" ## blue::user_data
colorLabels = color_df['color'].to_dict()
if Debug:
print(color_df)
print(colorLabels)
## parse results
if options.project:
outdir_report = HCGB_files.create_subfolder("report", outdir)
#final_dir = outdir + '/report/cluster'
final_dir = functions.create_subfolder("cluster", outdir_report)
else:
final_dir = outdir
## compare
name = 'cluster_' + str(HCGB_time.create_human_timestamp())
tag_cluster_info = final_dir + '/' + name
print('+ Saving results in folder: ', final_dir)
print('\tFile name: ', name)
(DataMatrix, labeltext) = min_hash_caller.compare(siglist_all,
tag_cluster_info, Debug)
## get colorLabels
## plot images
pdf = True
cluster_returned = min_hash_caller.plot(DataMatrix, labeltext,
tag_cluster_info, pdf, colorLabels)
## generate newick tree
min_hash_caller.get_Newick_tree(cluster_returned, DataMatrix, labeltext,
tag_cluster_info)
return ()
def run_prep(options):
"""
Main function of the prep module.
This module prepares fastq files for later usage. It initially checks the length
of the name and advises the user to rename samples if exceeded. Along ``BacterialTyper``
there are a few string length limitations by different software that need to be sort
out from the beginning of the process.
This module allows to user to copy files into the project folder initiate or only link using
a symbolic link to avoid duplicated raw data.
See additional details of this module in user_guide :ref:`prep module entry<prep-description>`.
.. seealso:: This function depends on other HCGB functions called:
- :func:`HCGB.sampleParser`
- :func:`HCGB.functions.aesthetics_functions`
- :func:`HCGB.functions.time_functions`
- :func:`HCGB.functions.main_functions`
- :func:`HCGB.functions.file_functions`
"""
## help_format option
if (options.help_format):
help_info.help_fastq_format()
exit()
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Preparing samples")
print ("--------- Starting Process ---------")
HCGB_time.print_time()
## init time
start_time_total = time.time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = os.path.abspath(options.output_folder)
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## Project mode as default
project_mode=True
if (options.detached):
options.project = False
project_mode=False
else:
options.project = True
## output folder
print ("\n+ Create output folder(s):")
HCGB_files.create_folder(outdir)
### info
final_dir = ""
if (options.project):
print ("+ Generate a directory containing information within the project folder provided")
final_dir = HCGB_files.create_subfolder("info", outdir)
else:
final_dir = outdir
## get files
pd_samples_retrieved = sampleParser.files.get_files(options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug)
## Information returned in pd_samples_retrieved
### sample, dirname, name, name_len, lane, read_pair, lane_file, ext, gz
if options.debug:
HCGB_aes.debug_message("pd_samples_retrieved", "yellow")
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## check character limitation
list_lengths = pd_samples_retrieved.loc[:,'name_len'].to_list()
if any(i > 10 for i in list_lengths):
print (colored("\t ** Name lengths exceeds the 10 character limitation...", 'yellow'))
if not (options.rename):
print (colored("** ERROR: Rename files or provide --rename option...", 'red'))
exit()
### rename files
if (options.rename):
options.rename = os.path.abspath(options.rename)
if not HCGB_files.is_non_zero_file(options.rename):
print (colored("** ERROR: File provided with rename information is not readable.", 'red'))
print (options.rename)
exit()
names_retrieved = pd.read_csv(options.rename, sep=',',
index_col=0, squeeze=True,
header=None).to_dict() ## read csv to dictionary
if (options.debug):
HCGB_aes.debug_message("names_retrieved", "yellow")
print (names_retrieved)
## TODO: check integrity of new names and special characters
## print to a file
timestamp = time_functions.create_human_timestamp()
rename_details = final_dir + '/' + timestamp + '_prep_renameDetails.txt'
rename_details_hd = open(rename_details, 'w')
## rename files
for index, row in pd_samples_retrieved.iterrows():
if (row['gz']):
extension_string = row['ext'] + row['gz']
else:
extension_string = row['ext']
if options.single_end:
renamed = names_retrieved[row['name']] + '.' + extension_string
else:
renamed = names_retrieved[row['name']] + '_' + row['read_pair'] + '.' + extension_string
## modify frame
pd_samples_retrieved.loc[index, 'new_file'] = renamed
pd_samples_retrieved.loc[index, 'new_name'] = names_retrieved[row['name']]
## save in file
string = row['sample'] + '\t' + renamed + '\n'
rename_details_hd.write(string)
if (options.debug):
print (colored('** DEBUG: rename', 'yellow'))
print ("Original: ", row['name'])
print ("Renamed: ", names_retrieved[row['name']])
print ("File:", renamed)
rename_details_hd.close()
##elif (options.single_end): It should work for both
print ("+ Sample files have been renamed...")
else:
pd_samples_retrieved['new_file'] = pd_samples_retrieved['file']
## create outdir for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "raw", options.debug)
## merge option
if (options.merge):
print ("+ Sample files will be merged...")
## TODO: check when rename option provided
pd_samples_merged = sampleParser.merge.one_file_per_sample(
pd_samples_retrieved, outdir_dict, options.threads,
final_dir, options.debug)
if (options.rename):
print ("+ Merge files have been renamed...")
else:
print ("+ Sample files have been merged...")
## process is finished here
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting prep module.")
exit()
## debugging messages
if (options.debug):
print (colored("** DEBUG: pd_samples_retrieved", 'yellow'))
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
print (colored("** DEBUG: outdir_dict", 'yellow'))
print (outdir_dict)
## copy or create symbolic link for files
if (options.copy):
print ("+ Sample files will be copied...")
## print to a file
timestamp = HCGB_time.create_human_timestamp()
copy_details = final_dir + '/' + timestamp + '_prep_copyDetails.txt'
copy_details_hd = open(copy_details, 'w')
else:
print ("+ Sample files will be linked...")
list_reads = []
for index, row in pd_samples_retrieved.iterrows():
if (options.copy):
## TODO: debug & set threads to copy faster
shutil.copy(row['sample'], os.path.join(outdir_dict[row['new_name']], row['new_file'] ))
string = row['sample'] + '\t' + os.path.join(outdir_dict[row['new_name']], row['new_file']) + '\n'
copy_details_hd.write(string)
else:
list_reads.append(row['new_file'])
if options.project:
HCGB_files.get_symbolic_link_file(row['sample'],
os.path.join(outdir_dict[row['new_name']], row['new_file']))
if (options.copy):
print ("+ Sample files have been copied...")
copy_details_hd.close()
else:
if not options.project:
HCGB_files.get_symbolic_link(list_reads, outdir)
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting prep module.")
return()
