def module_call(db_folder, dictionary_fasta_files, debug):
"""
"""
files_functions.create_folder(db_folder)
if db_folder.endswith("spaTyper"):
spaTyper_db = db_folder
else:
spaTyper_db = os.path.join(db_folder, "spaTyper")
files_functions.create_folder(spaTyper_db)
## check if files are available
(spaTyper_repeats, spaTyper_types) = check_files(spaTyper_db, debug)
## Get the SpaTypes in fasta sequences
seqDict, letDict, typeDict, seqLengths = spaTyper.spa_typing.getSpaTypes(
spaTyper_repeats, spaTyper_types, debug)
## debug messages
if debug:
print(
'## Debug: seqDict: Too large to print: See repeat_file for details'
)
print(
'## Debug: typeDict: Too large to print: See repeat_order_file for details'
)
print('## Debug: letDict: conversion dictionary')
print(letDict)
print('## Debug: seqLengths:')
print(seqLengths)
## summary results
results_summary = pd.DataFrame(columns=("sample", "sequence", "Repeats",
"Repeat Type"))
## for each sample get spaType
for key, value in dictionary_fasta_files.items():
print("+ Sample: ", key)
returned_value = call_spaTyper(value, seqDict, letDict, typeDict,
seqLengths, debug)
if len(returned_value.keys()) > 1:
print(
colored(
"** Attention: >1 spaTypes detected for sample: %s" % key,
'red'))
for j in returned_value.keys():
splitted = returned_value[j].split('::')
results_summary.loc[len(results_summary)] = (key, j, splitted[2],
splitted[1])
## debug messages
if debug:
print("Sequence name: ", j, "Repeats:", splitted[2],
"Repeat Type:", splitted[1], '\n')
##
return (results_summary)
def run_module_SPADES_old(name, folder, file1, file2, threads):
print ("+ Calling spades assembly for sample...", name)
## folder create
HCGB_files.create_folder(folder)
## get configuration
SPADES_bin = set_config.get_exe('spades')
## assembly main
path_to_contigs = run_SPADES_assembly(folder, file1, file2, name, SPADES_bin, threads)
## assembly plasmids
path_to_plasmids = run_SPADES_plasmid_assembly(folder, file1, file2, name, SPADES_bin, threads)
## discard plasmids from main
(tmp_contigs, tmp_plasmids) = discardPlasmids(path_to_contigs, path_to_plasmids, folder, name)
## rename fasta sequences
new_contigs = tmp_contigs.split(".fna.tmp")[0] + '.fna'
rename_contigs(tmp_contigs, "scaffolds_chr", new_contigs)
new_plasmids=""
if os.path.isfile(tmp_plasmids):
new_plasmids = tmp_plasmids.split(".fna.tmp")[0] + '.fna'
rename_contigs(tmp_plasmids, "scaffolds_plasmids", new_plasmids)
## contig stats
stats(new_contigs, new_plasmids)
## success stamps
filename_stamp = folder + '/.success'
stamp = HCGB_time.print_time_stamp(filename_stamp)
def ariba_pubmlstget(species, outdir):
######################################################################################
## usage: ariba pubmlstget [options] <"species in quotes"> <output_directory>
######################################################################################
## Download typing scheme for a given species from PubMLST, and make an ARIBA db
## positional arguments:
## species Species to download. Put it in quotes
## outdir Name of output directory to be made (must not already exist)
######################################################################################
## download information in database folder provided by config
print ("+ Call ariba module 'pubmlstget' to retrieve MLST information.")
HCGB_files.create_folder(outdir)
cmd = 'ariba pubmlstget "%s" %s' %(species, outdir)
return(HCGB_sys.system_call(cmd))
def R_package_path_installed():
"""Provides absolute path to file ``R_package.info.txt`` containing path to missing R packages installed"""
## check if exists or try to install
RDir_package = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'R', 'R_package.info.txt')
if HCGB_files.is_non_zero_file(RDir_package):
list = HCGB_main.readList_fromFile(RDir_package)
return (list[0])
else:
path2install = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'R',
'install_packages')
HCGB_files.create_folder(path2install)
return (path2install)
def download_VFDB_files(folder):
##
## Given a folder, check if it contains VFDB information
## or download it from website: http://www.mgc.ac.cn
##
links = (
"http://www.mgc.ac.cn/VFs/Down/VFs.xls.gz",
"http://www.mgc.ac.cn/VFs/Down/Comparative_tables_from_VFDB.tar.gz")
## check if data is downloaded, how old is the data and if it is necessary to download again
## consider >30 days long enough to be updated again
## time stamp
filename_stamp = folder + '/download_timestamp.txt'
if os.path.exists(folder):
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print("+ A previous download generated results on: ", stamp)
days_passed = HCGB_time.get_diff_time(filename_stamp)
print("\t\t** %s days ago" % days_passed)
if (days_passed > 30): ## download again
print(
"\t\t** Downloading information again just to be sure...")
else:
print("\t\t** No need to download data again.")
return ()
else:
HCGB_files.create_folder(folder)
## Open file and readlines
print('+ Downloading files:\n')
for line in links:
if not line.startswith('#'):
HCGB_sys.wget_download(line, folder)
## decompress files
print('+ Decompressing gzip files\n')
files = os.listdir(folder)
for item in files:
#print (folder)
if item.endswith('.gz'):
HCGB_files.extract(folder + '/' + item, folder)
## make stamp time
HCGB_time.print_time_stamp(filename_stamp)
return ()
def install_R_packages(package, source, install_path, extra):
(install_R, install_github_package) = get_install_R_files()
HCGB_files.create_folder(install_path)
Rscript_exe = set_config.get_exe('Rscript')
print("+ Installing %s package..." %package)
install_file = install_R
if (source == 'github'):
install_file = install_github_package
package= extra + '/' + package
cmd_R = '%s %s -l %s -p %s' %(Rscript_exe, install_file, package, install_path)
HCGB_sys.system_call(cmd_R)
## check if exists or try to install
MLSTar_package = os.path.join(install_path, 'MLSTar')
if os.path.exists(MLSTar_package):
RDir_package = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'R', 'R_package.info.txt')
HCGB_main.printList2file(RDir_package, [install_path])
else:
print_error_message(package, "No R package found", 'package')
print ('Please install manually to proceed...')
def main():
## control if options provided or help
if len(sys.argv) > 1:
print ("")
else:
help_options()
exit()
name = argv[1]
fasta_file = os.path.abspath(argv[2])
folder = os.path.abspath(argv[3])
debug=True
## path
folder = HCGB_files.create_folder(folder)
## ATTENTION: agrvate needs to chdir to output folder
os.chdir(folder)
###
agrvate_call(name, fasta_file, folder, debug)
def agrvate_caller(dict_assemblies, dict_folders, debug=False):
"""Create agrvate call and control for parameters"""
## ATTENTION: agrvate needs to chdir to output folder
path_here = os.getcwd()
print ("+ Checking agr genes for each sample retrieved...")
agrvate_results = pd.DataFrame()
## No need to optimize. There is a problem with the working dir of agrvate and we
## need to change every time.
for name, assembly_file in dict_assemblies.items():
sample_folder = HCGB_files.create_folder(dict_folders[name])
## check if previously done and succeeded
filename_stamp = sample_folder + '/.success'
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print (colored("\tA previous command generated results on: %s [%s]" %(stamp, name), 'yellow'))
else:
os.chdir(sample_folder)
info_sample = agrvate_call(name, assembly_file, sample_folder, debug)
agrvate_results = pd.concat([agrvate_results, info_sample], join='outer')
if (info_sample.shape[0] == 0):
print("+ Some error occurred with sample %s. Please re-run analysis or check log files." %name)
else:
## success
HCGB_time.print_time_stamp(filename_stamp)
print ("+ Jobs finished%s\n+ Collecting information for all samples...")
os.chdir(path_here)
## debug messages
if debug:
HCGB_aes.debug_message('agrvate_results', 'yellow')
HCGB_main.print_all_pandaDF(agrvate_results)
return(agrvate_results)
def index_database(fileToIndex, kma_bin, index_name, option, folder, type_option):
"""
Calls KMA_ software to index fasta files into a database for later KMA identification.
:param fileToIndex: Fasta file to include in the database.
:param kma_bin: Absolute path to kma executable binary.
:param index_name: Name for the database.
:param option: Option to create or update a database.
:param folder: Absolute path to folder containing database.
:param type_option: Option to index the database with batch: batch [Default: Off].
:type fileToIndex: string
:type kma_bin: string
:type index_name: string
:type option: string
:type folder: string
:type type_option: string
:returns: It returns message from :func:`BacterialTyper.scripts.species_identification_KMA.check_db_indexed`.
.. seealso:: This function depends on other ``BacterialTyper`` functions called:
- :func:`BacterialTyper.scripts.functions.create_folder`
- :func:`BacterialTyper.scripts.functions.system_call`
- :func:`BacterialTyper.scripts.species_identification_KMA.check_db_indexed`
"""
########################################################################################
## KMA_index-1.2.2
########################################################################################
# kma_index creates the databases needed to run KMA, from a list of fasta files given.
# Options are:
# Desc: Default:
#
# -i Input/query file name (STDIN: "--") None
# -o Output file Input/template file
# -batch Batch input file
# -deCon File with contamination (STDIN: "--") None/False
# -batchD Batch decon file
# -t_db Add to existing DB None/False
# -k Kmersize 16
# -k_t Kmersize for template identification 16
# -k_i Kmersize for indexing 16
# -ML Minimum length of templates kmersize (16)
# -CS Start Chain size 1 M
# -ME Mega DB False
# -NI Do not dump *.index.b False
# -Sparse Make Sparse DB ('-' for no prefix) None/False
# -ht Homology template 1.0
# -hq Homology query 1.0
# -and Both homolgy thresholds
# has to be reached or
# -v Version
# -h Shows this help message
#######################################################################################
## check if file exists
if os.path.isfile(index_name):
index_file_name = index_name
else:
index_file_name = folder + '/' + index_name
logFile = index_file_name + '.log'
## check if folder exists
HCGB_files.create_folder(folder)
## single file
if (type_option == 'batch'):
type_option = '-batch'
else:
type_option = '-i'
## new or add to existing db
if (option == "new"):
print ("\n+ Generate and index database for kmer alignment search...\n")
cmd_kma_index = "%s index %s %s -o %s 2> %s" %(kma_bin, type_option, fileToIndex, index_file_name, logFile)
elif (option == "add"):
print ("\n+ Updating database with new entries...\n")
cmd_kma_index = "%s index %s %s -o %s -t_db %s 2> %s" %(kma_bin, type_option, fileToIndex, index_file_name, index_file_name, logFile)
code = HCGB_sys.system_call(cmd_kma_index)
if code == 'FAIL':
print (colored("Database generated an error during the index: %s" %index_name, 'red'))
print (colored("EXIT", 'red'))
exit()
return_code = check_db_indexed(index_file_name, folder)
return(return_code)
def BUSCO_check(input_dir, outdir, options, start_time_total, mode):
HCGB_aes.boxymcboxface("BUSCO Analysis Quality check")
## absolute path for in & out
database_folder = os.path.abspath(options.database)
## get files and get dir for each sample according to mode
if mode == 'genome':
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
if not options.project:
outdir = HCGB_files.create_subfolder("assembly_qc", outdir)
if options.debug:
print("** DEBUG: pd_samples_retrieved")
print(pd_samples_retrieved)
BUSCO_outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved,
"assemble_qc",
options.debug)
elif mode == 'proteins':
pd_samples_retrieved = sampleParser.files.get_files(
options, outdir, "annot", ["faa"], options.debug) ##
if not options.project:
outdir = HCGB_files.create_subfolder("annot_qc", outdir)
if options.debug:
print("** DEBUG: pd_samples_retrieved")
print(pd_samples_retrieved)
BUSCO_outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved,
"annot_qc",
options.debug)
## add column to dataframe
pd_samples_retrieved['busco_folder'] = ""
for index, row in pd_samples_retrieved.iterrows():
pd_samples_retrieved.at[index, 'busco_folder'] = BUSCO_outdir_dict[
row['name']]
## debug message
if (options.debug):
HCGB_aes.debug_message("df_samples_busco", 'yellow')
print(pd_samples_retrieved)
HCGB_aes.debug_message("BUSCO_outdir_dict", 'yellow')
print(BUSCO_outdir_dict)
## Check each using BUSCO
database_folder = os.path.abspath(options.database)
BUSCO_Database = HCGB_files.create_subfolder('BUSCO', database_folder)
if not os.path.exists(BUSCO_Database):
HCGB_files.create_folder(BUSCO_Database)
## call
(dataFrame_results, stats_results) = BUSCO_caller.BUSCO_call(
options.BUSCO_dbs, pd_samples_retrieved, BUSCO_Database,
options.threads, mode)
## debug message
if (options.debug):
HCGB_aes.debug_message("dataFrame_results", 'yellow')
HCGB_main.print_all_pandaDF(dataFrame_results)
## functions.timestamp
print("+ Quality control of all samples finished: ")
start_time_partial = HCGB_time.timestamp(start_time_total)
## multiqc report plot
if (options.skip_report):
print("+ No report generation...")
else:
print("\n+ Generating a report BUSCO plot.")
outdir_report = HCGB_files.create_subfolder("report", outdir)
## get subdirs generated and call multiQC report module
givenList = []
print(
"+ Detail information for each sample could be identified in separate folders."
)
## name folder according to mode
if mode == 'genome':
BUSCO_report = HCGB_files.create_subfolder("BUSCO_assembly",
outdir_report)
elif mode == 'proteins':
BUSCO_report = HCGB_files.create_subfolder("BUSCO_annot",
outdir_report)
## generate plots
print("+ Generate summarizing plots...")
BUSCO_caller.BUSCO_plots(dataFrame_results, BUSCO_report,
options.threads)
print('\n+ Check quality plots in folder: %s' % BUSCO_report)
## TODO
## Parse BUSCO statistics in dataframe (stats_results) for discarding samples if necessary
## given a cutoff, discard or advise to discard some samples
### print statistics
stats_results.to_csv(BUSCO_report + "/BUSCO_stats.csv")
name_excel = BUSCO_report + "/BUSCO_stats.xlsx"
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
stats_results.to_excel(writer, sheet_name="BUSCO statistics")
writer.save()
print('\n+ Check quality statistics in folder: %s' % BUSCO_report)
return (dataFrame_results)
def run_phylo(options):
"""
Main function acting as an entry point to the module *phylo*.
"""
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Phylogenetic reconstruction")
print ("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir=""
## set mode: project/detached
## Project mode as default
project_mode=True
if (options.detached):
options.project = False
project_mode=False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get the database
options.database = os.path.abspath(options.database)
### parse the reference
print ("+ Retrieve the reference...")
reference_gbk_file = get_reference_gbk(options)
## generate output folder, if necessary
print ("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
##################################
## select samples and map
####################################
print ("+ Retrieve samples to map available...")
dict_folders = map_samples(options, reference_gbk_file, input_dir, outdir)
if Debug:
print (colored("**DEBUG: dict_folders **", 'yellow'))
print (dict_folders)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
##################################
## Create core alingment
##################################
outdir_report = HCGB_files.create_subfolder("report", outdir)
phylo_dir = HCGB_files.create_subfolder("phylo", outdir_report)
analysis_dir = HCGB_files.create_subfolder(options.name, phylo_dir)
snippy_dir = HCGB_files.create_subfolder("snippy", analysis_dir)
list_folders = list(dict_folders.values())
options_string = ""
variant_calling.snippy_core_call(list_folders, options_string, options.name,
snippy_dir, options.output_format, Debug)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## snp distance matrix
snp_distance_dir = HCGB_files.create_subfolder("snp_distance", analysis_dir)
name_matrix = os.path.join(snp_distance_dir, "snp_matrix_" + options.name)
countGaps = False
aln_file = os.path.join(snippy_dir, options.name + '.aln')
phylo_parser.get_snp_distance(aln_file, options.output_format, countGaps, name_matrix, Debug)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## phylogenetic analysis
iqtree_output = HCGB_files.create_subfolder("iqtree", analysis_dir)
phylo_parser.ml_tree(snippy_dir, options.name, options.threads, iqtree_output, Debug)
## time stamp
start_time_partial = HCGB_files.timestamp(start_time_total)
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting Annotation module.")
return()
def run_prep(options):
"""
Main function of the prep module.
This module prepares fastq files for later usage. It initially checks the length
of the name and advises the user to rename samples if exceeded. Along ``BacterialTyper``
there are a few string length limitations by different software that need to be sort
out from the beginning of the process.
This module allows to user to copy files into the project folder initiate or only link using
a symbolic link to avoid duplicated raw data.
See additional details of this module in user_guide :ref:`prep module entry<prep-description>`.
.. seealso:: This function depends on other HCGB functions called:
- :func:`HCGB.sampleParser`
- :func:`HCGB.functions.aesthetics_functions`
- :func:`HCGB.functions.time_functions`
- :func:`HCGB.functions.main_functions`
- :func:`HCGB.functions.file_functions`
"""
## help_format option
if (options.help_format):
help_info.help_fastq_format()
exit()
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Preparing samples")
print ("--------- Starting Process ---------")
HCGB_time.print_time()
## init time
start_time_total = time.time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = os.path.abspath(options.output_folder)
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## Project mode as default
project_mode=True
if (options.detached):
options.project = False
project_mode=False
else:
options.project = True
## output folder
print ("\n+ Create output folder(s):")
HCGB_files.create_folder(outdir)
### info
final_dir = ""
if (options.project):
print ("+ Generate a directory containing information within the project folder provided")
final_dir = HCGB_files.create_subfolder("info", outdir)
else:
final_dir = outdir
## get files
pd_samples_retrieved = sampleParser.files.get_files(options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"), options.debug)
## Information returned in pd_samples_retrieved
### sample, dirname, name, name_len, lane, read_pair, lane_file, ext, gz
if options.debug:
HCGB_aes.debug_message("pd_samples_retrieved", "yellow")
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## check character limitation
list_lengths = pd_samples_retrieved.loc[:,'name_len'].to_list()
if any(i > 10 for i in list_lengths):
print (colored("\t ** Name lengths exceeds the 10 character limitation...", 'yellow'))
if not (options.rename):
print (colored("** ERROR: Rename files or provide --rename option...", 'red'))
exit()
### rename files
if (options.rename):
options.rename = os.path.abspath(options.rename)
if not HCGB_files.is_non_zero_file(options.rename):
print (colored("** ERROR: File provided with rename information is not readable.", 'red'))
print (options.rename)
exit()
names_retrieved = pd.read_csv(options.rename, sep=',',
index_col=0, squeeze=True,
header=None).to_dict() ## read csv to dictionary
if (options.debug):
HCGB_aes.debug_message("names_retrieved", "yellow")
print (names_retrieved)
## TODO: check integrity of new names and special characters
## print to a file
timestamp = time_functions.create_human_timestamp()
rename_details = final_dir + '/' + timestamp + '_prep_renameDetails.txt'
rename_details_hd = open(rename_details, 'w')
## rename files
for index, row in pd_samples_retrieved.iterrows():
if (row['gz']):
extension_string = row['ext'] + row['gz']
else:
extension_string = row['ext']
if options.single_end:
renamed = names_retrieved[row['name']] + '.' + extension_string
else:
renamed = names_retrieved[row['name']] + '_' + row['read_pair'] + '.' + extension_string
## modify frame
pd_samples_retrieved.loc[index, 'new_file'] = renamed
pd_samples_retrieved.loc[index, 'new_name'] = names_retrieved[row['name']]
## save in file
string = row['sample'] + '\t' + renamed + '\n'
rename_details_hd.write(string)
if (options.debug):
print (colored('** DEBUG: rename', 'yellow'))
print ("Original: ", row['name'])
print ("Renamed: ", names_retrieved[row['name']])
print ("File:", renamed)
rename_details_hd.close()
##elif (options.single_end): It should work for both
print ("+ Sample files have been renamed...")
else:
pd_samples_retrieved['new_file'] = pd_samples_retrieved['file']
## create outdir for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project, pd_samples_retrieved, "raw", options.debug)
## merge option
if (options.merge):
print ("+ Sample files will be merged...")
## TODO: check when rename option provided
pd_samples_merged = sampleParser.merge.one_file_per_sample(
pd_samples_retrieved, outdir_dict, options.threads,
final_dir, options.debug)
if (options.rename):
print ("+ Merge files have been renamed...")
else:
print ("+ Sample files have been merged...")
## process is finished here
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting prep module.")
exit()
## debugging messages
if (options.debug):
print (colored("** DEBUG: pd_samples_retrieved", 'yellow'))
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
print (colored("** DEBUG: outdir_dict", 'yellow'))
print (outdir_dict)
## copy or create symbolic link for files
if (options.copy):
print ("+ Sample files will be copied...")
## print to a file
timestamp = HCGB_time.create_human_timestamp()
copy_details = final_dir + '/' + timestamp + '_prep_copyDetails.txt'
copy_details_hd = open(copy_details, 'w')
else:
print ("+ Sample files will be linked...")
list_reads = []
for index, row in pd_samples_retrieved.iterrows():
if (options.copy):
## TODO: debug & set threads to copy faster
shutil.copy(row['sample'], os.path.join(outdir_dict[row['new_name']], row['new_file'] ))
string = row['sample'] + '\t' + os.path.join(outdir_dict[row['new_name']], row['new_file']) + '\n'
copy_details_hd.write(string)
else:
list_reads.append(row['new_file'])
if options.project:
HCGB_files.get_symbolic_link_file(row['sample'],
os.path.join(outdir_dict[row['new_name']], row['new_file']))
if (options.copy):
print ("+ Sample files have been copied...")
copy_details_hd.close()
else:
if not options.project:
HCGB_files.get_symbolic_link(list_reads, outdir)
print ("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print ("+ Exiting prep module.")
return()
def run(options):
"""
This is the main function of the module ``config``. It basically checks
if the different requirements (python` and third-party software) are
fulfilled.
If any requirement is not available this modules tries to install them or reports to the user to
manually install them.
:param option: State whether to check or install missing modules, packages and third party software. Provide: check/install
:param install_path: Absolute path to install modules or packages missing. Default: ``BacterialTyper`` environment path.
:param IslandPath: True/False for checking additional perl and software required by this option analysis.
:param debug: True/false for debugging messages.
:type option: string
:type IslandPath: boolean
:type install_path: string
:type debug: boolean
.. seealso:: This function depends on several ``BacterialTyper`` functions:
- :func:`BacterialTyper.config.set_config.check_python_packages`
- :func:`BacterialTyper.config.set_config.check_perl_packages`
- :func:`BacterialTyper.config.extern_progs.return_min_version_soft`
- :func:`BacterialTyper.config.extern_progs.print_dependencies`
"""
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Pipeline Configuration")
print("--------- Starting Process ---------")
HCGB_time.print_time()
if (options.install_path):
if os.path.isdir(options.install_path):
if (Debug):
print(
"Installation path provided for missing modules, packages, dependencies..."
)
print("Path: " + options.install_path)
else:
print(colored("\n*** ERROR ****", 'red'))
print(colored("Path provided is not a folder", 'red'))
print(options.install_path)
exit()
else:
## get python environment path
env_bin_directory = os.path.dirname(os.environ['_'])
##os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'templates'))
options.install_path = os.path.abspath(
os.path.join(env_bin_directory, '../software'))
if (Debug):
print("Retrieve environment path as installation path:")
print("Path: " + options.install_path)
HCGB_files.create_folder(options.install_path)
#######################
## install or only check
#######################
option_install = False
if (options.option == 'install'):
print("\n+ Check dependencies")
print(
"+ Try to install all missing dependencies, modules or third party software..."
)
option_install = True
## check if access and permission
if os.path.isdir(options.install_path):
if (set_config.access_check(options.install_path, mode=os.F_OK)):
print(
"Installation path is accessible and has permission for installation if necessary"
)
else:
print(colored("\n*** ERROR ****", 'red'))
print(
colored(
"No access/permission for this path: %s" %
options.install_path, 'red'))
print(
colored(
"Please provide a valid path with access/permission to install any missing dependencies.",
'red'))
exit()
else:
print(colored("\n*** ERROR ****", 'red'))
print(colored("Path provided is not a folder", 'red'))
print(options.install_path)
exit()
elif (options.option == 'only_check'):
print(
"\nCheck dependencies, modules or third party software and print report..."
)
#######################
## python version
#######################
HCGB_aes.print_sepLine("+", 20, False)
print('Python:')
HCGB_aes.print_sepLine("+", 20, False)
this_python_version = str(sys.version)
python_min_version = extern_progs.return_min_version_soft('python')
if LooseVersion(this_python_version) >= LooseVersion(python_min_version):
print(
colored(
"Minimum version (%s) satisfied: %s" %
(python_min_version, this_python_version), 'green'))
else:
print(
colored(
"Minimum version (%s) not satisfied: %s" %
(python_min_version, this_python_version), 'red'))
exit()
#######################
## perl_version
#######################
print('\n')
HCGB_aes.print_sepLine("+", 50, False)
print('Perl:')
HCGB_aes.print_sepLine("+", 50, False)
perl_min_version = extern_progs.return_min_version_soft('perl')
this_perl_path = set_config.get_exe("perl", Debug)
this_perl_version = set_config.get_version("perl", this_perl_path, Debug)
if LooseVersion(this_perl_version) >= LooseVersion(perl_min_version):
print(
colored(
"Minimum version (%s) satisfied: %s" %
(perl_min_version, this_perl_version), 'green'))
else:
print(
colored(
"Minimum version (%s) not satisfied: %s" %
(perl_min_version, this_perl_version), 'red'))
exit()
#######################
## third-party software
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('External dependencies:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_dependencies(option_install, options.install_path, Debug)
print('\n')
#######################
## python packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('Python packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_python_packages(Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## perl packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('Perl packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_perl_packages("perl_dependencies", Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## IslandPath dependencies
#######################
if (options.IslandPath):
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('IslandPath packages and software required:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_IslandPath(Debug, option_install,
options.install_path)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
#######################
## R packages
#######################
print('\n')
HCGB_aes.print_sepLine("+", 20, False)
print('R packages:')
HCGB_aes.print_sepLine("+", 20, False)
set_config.check_R_packages(option_install, options.install_path, Debug)
HCGB_aes.print_sepLine("+", 20, False)
print('\n')
def biotype_all(featureCount_exe, path, gtf_file, bam_file, name, threads, Debug, allow_multimap, stranded):
## folder for results
if not os.path.isdir(path):
files_functions.create_folder(path)
out_file = os.path.join(path, 'featureCount.out')
logfile = os.path.join(path, name + '_RNAbiotype.log')
filename_stamp_all = path + '/.success_all'
if os.path.isfile(filename_stamp_all):
stamp = time_functions.read_time_stamp(filename_stamp_all)
print (colored("\tA previous command generated results on: %s [%s -- %s]" %(stamp, name, 'RNAbiotype'), 'yellow'))
return()
else:
filename_stamp_featureCounts = path + '/.success_featureCounts'
if os.path.isfile(filename_stamp_featureCounts):
stamp = time_functions.read_time_stamp(filename_stamp_featureCounts)
print (colored("\tA previous command generated results on: %s [%s -- %s]" %(stamp, name, 'featureCounts'), 'yellow'))
else:
## debugging messages
if Debug:
print ("** DEBUG:")
print ("featureCounts system call for sample: " + name)
print ("out_file: " + out_file)
print ("logfile: " + logfile)
## send command for feature count
## Allow multimapping
if allow_multimap:
cmd_featureCount = ('%s -s %s -M -O -T %s -p -t exon -g transcript_biotype -a %s -o %s %s 2> %s' %(
featureCount_exe, stranded, threads, gtf_file, out_file, bam_file, logfile)
)
else:
cmd_featureCount = ('%s -s %s --largestOverlap -T %s -p -t exon -g transcript_biotype -a %s -o %s %s 2> %s' %(
featureCount_exe, stranded, threads, gtf_file, out_file, bam_file, logfile)
)
## system call
cmd_featureCount_code = system_call_functions.system_call(cmd_featureCount, False, True)
if not cmd_featureCount_code:
print("** ERROR: featureCount failed for sample " + name)
exit()
## print time stamp
time_functions.print_time_stamp(filename_stamp_featureCounts)
## parse results
(extended_Stats_file, RNAbiotypes_stats_file) = parse_featureCount(out_file, path, name, bam_file, Debug)
## debugging messages
if Debug:
print ("** DEBUG:")
print ("extended_Stats: " + extended_Stats_file)
print (main_functions.get_data(extended_Stats_file, '\t', 'header=None'))
print ("RNAbiotypes_stats: " + RNAbiotypes_stats_file)
print (main_functions.get_data(RNAbiotypes_stats_file, '\t', 'header=None'))
return ()
def run_profile(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
if (options.help_project):
## information for project
help_info.project_help()
exit()
if (options.help_ARIBA):
## help_format option
ariba_caller.help_ARIBA()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Virulence & Resistance profile module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
options.database = os.path.abspath(options.database)
global input_dir
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
global Project
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "profile",
options.debug)
###
print(
"+ Generate a sample profile for virulence and resistance candidate genes for each sample retrieved using:"
)
print(
"(1) Antimicrobial Resistance Inference By Assembly (ARIBA) software")
print(
"(2) Pre-defined databases by different suppliers or user-defined databases."
)
## get databases to check
retrieve_databases = get_options_db(options)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_total)
########
ARIBA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases,
start_time_partial)
######################################
## update database for later usage
######################################
if not options.fast:
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
HCGB_aes.boxymcboxface("Update Sample Database")
## update db
print("+ Update database with samples identified")
## TODO: check if it works
dataBase_user = database_user.update_database_user_data(
options.database, input_dir, Debug, options)
## debug message
if (Debug):
print(colored("**DEBUG: results obtained **", 'yellow'))
else:
print(
"+ No update of the database has been requested using option --fast"
)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Virulence & Resistance profile module.")
return ()
def run_biotype(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_XICRA.help_fastq_format()
elif (options.help_project):
## information for project
help_XICRA.project_help()
exit()
elif (options.help_RNAbiotype):
## information for join reads
RNAbiotype.help_info()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
aesthetics_functions.pipeline_header('XICRA')
aesthetics_functions.boxymcboxface("RNA biotype analysis")
print("--------- Starting Process ---------")
time_functions.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
if (options.detached):
outdir = os.path.abspath(options.output_folder)
options.project = False
else:
options.project = True
outdir = input_dir
## get files
print('+ Getting files from input folder... ')
## get files
if options.noTrim:
print('+ Mode: fastq.\n+ Extension: ')
print("[ fastq, fq, fastq.gz, fq.gz ]\n")
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"),
options.debug)
else:
print('+ Mode: trim.\n+ Extension: ')
print("[ _trim_ ]\n")
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## Discard if joined reads: use trimmed single-end or paired-end
pd_samples_retrieved = pd_samples_retrieved[
pd_samples_retrieved['ext'] != '_joined']
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
files_functions.create_folder(outdir)
## for samples
mapping_outdir_dict = files_functions.outdir_project(
outdir, options.project, pd_samples_retrieved, "map", options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: mapping_outdir_dict **", 'yellow'))
print(mapping_outdir_dict)
# time stamp
start_time_partial = time_functions.timestamp(start_time_total)
## optimize threads
name_list = set(pd_samples_retrieved["new_name"].tolist())
threads_job = main_functions.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
##############################################
## map Reads
##############################################
start_time_partial = mapReads_module(options, pd_samples_retrieved,
mapping_outdir_dict, options.debug,
max_workers_int, threads_job,
start_time_partial, outdir)
## debug message
if (Debug):
print(colored("**DEBUG: mapping_results **", 'yellow'))
print(mapping_results)
# time stamp
start_time_partial = time_functions.timestamp(start_time_partial)
## for samples
biotype_outdir_dict = files_functions.outdir_project(
outdir, options.project, pd_samples_retrieved, "biotype",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: biotype_outdir_dict **", 'yellow'))
print(biotype_outdir_dict)
## get RNAbiotype information
RNAbiotype.RNAbiotype_module_call(mapping_results, biotype_outdir_dict,
options.annotation, options.debug,
max_workers_int, threads_job)
# time stamp
start_time_partial = time_functions.timestamp(start_time_partial)
if (options.skip_report):
print("+ No report generation...")
else:
print(
"\n+ Generating a report using MultiQC module for featureCount analysis."
)
outdir_report = files_functions.create_subfolder("report", outdir)
## get subdirs generated and call multiQC report module
givenList = []
print(
"+ Detail information for each sample could be identified in separate folders:"
)
## call multiQC report module
givenList = [v for v in biotype_outdir_dict.values()]
my_outdir_list = set(givenList)
## debug message
if (Debug):
print(
colored("\n**DEBUG: my_outdir_list for multiqc report **",
'yellow'))
print(my_outdir_list)
print("\n")
featureCount_report = files_functions.create_subfolder(
"featureCount", outdir_report)
multiQC_report.multiQC_module_call(my_outdir_list, "featureCount",
featureCount_report, "-dd 2")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% featureCount_report)
### Summarizing RNA biotype information
biotype_report = files_functions.create_subfolder(
"biotype", outdir_report)
single_files_biotype = files_functions.create_subfolder(
"samples", biotype_report)
## results
dict_files = {}
for samples in biotype_outdir_dict:
featurecount_file = os.path.join(biotype_outdir_dict[samples],
'featureCount.out.tsv')
if files_functions.is_non_zero_file(featurecount_file):
dict_files[samples] = featurecount_file
## copy pdf
pdf_plot = main_functions.retrieve_matching_files(
biotype_outdir_dict[samples], '.pdf', options.debug)
if files_functions.is_non_zero_file(pdf_plot[0]):
shutil.copy(pdf_plot[0], single_files_biotype)
## collapse all information
all_data = RNAbiotype.generate_matrix(dict_files)
## print into excel/csv
print('+ Table contains: ', len(all_data), ' entries\n')
## debugging messages
if Debug:
print("** DEBUG: all_data")
print(all_data)
## set abs_csv_outfile to be in report folder
## copy or link files for each sample analyzed
abs_csv_outfile = os.path.join(biotype_report, "summary.csv")
all_data.to_csv(abs_csv_outfile)
## create plot: call R [TODO: implement in python]
outfile_pdf = os.path.join(biotype_report, "RNAbiotypes_summary.pdf")
## R scripts
biotype_R_script = tools.R_scripts('plot_RNAbiotype_sum',
options.debug)
rscript = set_config.get_exe("Rscript", options.debug)
cmd_R_plot = "%s %s -f %s -o %s" % (rscript, biotype_R_script,
abs_csv_outfile, outfile_pdf)
##
print("+ Create summary plot for all samples")
callCode = system_call_functions.system_call(cmd_R_plot)
print("\n*************** Finish *******************")
start_time_partial = time_functions.timestamp(start_time_total)
print("\n+ Exiting join module.")
return ()
def run_assembly(options):
"""Main function of the assemble module.
It assembles each sample using SPADES_ and checks quality using BUSCO_ software and database.
.. seealso:: This function depends on other BacterialTyper and HCGB functions called:
- :func:`BacterialTyper.scripts.BUSCO_caller.print_help_BUSCO`
- :func:`BacterialTyper.scripts.multiQC_report.multiqc_help`
- :func:`BacterialTyper.modules.qc.BUSCO_check`
- :func:`HCGB.sampleParser`
- :func:`HCGB.functions.aesthetics_functions`
- :func:`HCGB.functions.time_functions`
- :func:`HCGB.functions.main_functions`
- :func:`HCGB.functions.file_functions`
.. include:: ../../links.inc
"""
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
exit()
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Assembly module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "assemble",
options.debug)
### call assemble using spades
start_time_partial = start_time_total
start_time_partial_assembly = start_time_partial
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
HCGB_aes.debug_message("options.threads: " + str(options.threads),
"yellow")
HCGB_aes.debug_message("max_workers: " + str(max_workers_int),
"yellow")
HCGB_aes.debug_message("cpu_here: " + str(threads_job), "yellow")
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
# We can use a with statement to ensure threads are cleaned up promptly
print('+ Running modules SPADES...')
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
## send for each sample
commandsSent = {
executor.submit(check_sample_assembly, name, outdir_dict[name],
sorted(cluster["sample"].tolist()), threads_job):
name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
## functions.timestamp
print("\n+ Assembly of all samples finished: ")
start_time_partial = HCGB_time.timestamp(start_time_partial_assembly)
##
if (assembly_stats):
###################
if Debug:
HCGB_aes.debug_message("assembly_stats dictionary", "yellow")
print(assembly_stats)
## create single file
get_assembly_stats_all(assembly_stats, outdir, Debug)
### symbolic links
print("+ Retrieve all genomes assembled...")
### BUSCO check assembly
if (options.no_BUSCO):
print()
else:
results = qc.BUSCO_check(outdir, outdir, options, start_time_partial,
"genome")
## print to file results
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Assembly module.")
return ()
def run_database(options):
## init time
start_time_total = time.time()
start_time_partial = start_time_total
## debugging messages
global Debug
if (options.debug):
Debug = True
print("[Debug mode: ON]")
else:
Debug = False
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Database")
print("--------- Starting Process ---------")
HCGB_time.print_time()
kma_bin = set_config.get_exe("kma")
######################################################
## print further information if requested
if (options.help_ARIBA):
print("ARIBA databases information:")
ariba_caller.help_ARIBA()
exit()
elif (options.help_BUSCO):
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_KMA):
species_identification_KMA.help_kma_database()
exit()
######################################################
## create folder
## absolute
options.path = os.path.abspath(options.path)
HCGB_files.create_folder(options.path)
#########
if Debug:
print(colored("DEBUG: absolute path folder: " + options.path,
'yellow'))
##########
## NCBI ##
##########
## if any NCBI options provided
if any([options.ID_file, options.descendant]):
## create folders
NCBI_folder = HCGB_files.create_subfolder('NCBI', options.path)
if (options.ID_file):
## get path and check if it is file
abs_path_file = os.path.abspath(options.ID_file)
if os.path.isfile(abs_path_file):
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check NCBI ids provided ---------\n")
HCGB_aes.print_sepLine("*", 70, False)
## get file information
print("\t+ Obtaining information from file: %s" %
abs_path_file)
strains2get = HCGB_main.get_data(abs_path_file, ',', '')
dataBase_NCBI = database_generator.NCBI_DB(
strains2get, NCBI_folder, Debug)
#########
if Debug:
print(colored("DEBUG: NCBI data provided: ", 'yellow'))
print(options.ID_file)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## strains downloaded would be included to a kma index
## Get all entries belonging to this taxon provided
if (options.descendant):
#########
if Debug:
print(colored("DEBUG: NCBI descendant option: ON ", 'yellow'))
print()
HCGB_aes.print_sepLine("*", 70, False)
print(
"--------- Check descendant NCBI taxonomy ids provided ---------\n"
)
HCGB_aes.print_sepLine("*", 70, False)
## [TODO]
dataBase_NCBI = database_generator.NCBI_descendant(
options.descendant, NCBI_folder, Debug)
##############################################################
## update KMA database with NCBI information retrieved
##############################################################
print('\n\n+ Update database for later identification analysis...')
list_of_files = dataBase_NCBI['genome'].tolist()
kma_db = HCGB_files.create_subfolder('KMA_db', options.path)
genbank_kma_db = HCGB_files.create_subfolder('genbank', kma_db)
print('+ Database to update: ', genbank_kma_db)
species_identification_KMA.generate_db(list_of_files, 'genbank_KMA',
genbank_kma_db, 'new', 'batch',
Debug, kma_bin)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
###############
## user_data ##
###############
if options.project_folder:
##
dataBase_user = pd.DataFrame()
## get absolute path
abs_project_folder = os.path.abspath(options.project_folder)
if os.path.exists(abs_project_folder):
#########
if Debug:
print(
colored("DEBUG: User provides folder containing project",
'yellow'))
print()
HCGB_aes.print_sepLine("*", 70, False)
print("--------- Check user provided project folder ---------")
HCGB_aes.print_sepLine("*", 70, False)
dataBase_user = database_user.update_database_user_data(
options.path, abs_project_folder, Debug, options)
else:
print(
colored(
"ERROR: Folder provided does not exists: %s" %
options.project_folder, 'red'))
exit()
##############################################################
## update KMA database with user_data information retrieved
##############################################################
print('\n\n+ Update database for later identification analysis...')
list_of_files = dataBase_user['genome'].tolist()
kma_db = HCGB_files.create_subfolder('KMA_db', options.path)
user_kma_db = HCGB_files.create_subfolder('user_data', kma_db)
print('+ Database to update: ', user_kma_db)
species_identification_KMA.generate_db(list_of_files, 'userData_KMA',
user_kma_db, 'new', 'batch',
Debug, kma_bin)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
##########
## ARIBA
##########
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check ARIBA parameters provided --------")
HCGB_aes.print_sepLine("*", 50, False)
if (options.no_ARIBA):
print("+ No ARIBA databases would be downloaded...")
#########
if Debug:
print(colored("DEBUG: No option ARIBA", 'yellow'))
else:
#functions.print_sepLine("*",50, False)
### ariba list databases
ariba_dbs_list = ['CARD', 'VFDB']
if (options.no_def_ARIBA):
ariba_dbs_list = options.ariba_dbs
else:
if (options.ariba_dbs):
ariba_dbs_list = ariba_dbs_list + options.ariba_dbs
ariba_dbs_list = set(ariba_dbs_list)
#########
if Debug:
print(colored("DEBUG: Option ARIBA", 'yellow'))
print(options.ariba_dbs)
ariba_caller.download_ariba_databases(ariba_dbs_list, options.path,
Debug, options.threads)
### ariba list databases
if (options.ariba_users_fasta):
print(
"+ Generate ARIBA database for databases provided: prepare fasta and metadata information"
)
#########
if Debug:
print(colored("DEBUG: Option user ARIBA db", 'yellow'))
print(ariba_users_fasta)
print(ariba_users_meta)
## [TODO]:
## ariba prepareref fasta and metadata
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
#########
## kma ##
#########
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check KMA parameters provided ----------")
kma_database = options.path + '/KMA_db'
HCGB_files.create_folder(kma_database)
## types: bacteria, archaea, protozoa, fungi, plasmids, typestrains
## downloads all "bacterial" genomes from KMA website
## kma: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/
print(
"+ Retrieving information from: ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder website"
)
## KMA databases to use
## only user dbs
if (options.no_def_kma):
if (options.kma_dbs):
print("+ Only user databases selected will be indexed...")
else:
print("+ No databases selected.")
print(colored("ERROR: Please select a kma database.", 'red'))
exit()
## default dbs + user
else:
kma_dbs = ["bacteria", "plasmids"]
## default dbs + user
if (options.kma_dbs):
options.kma_dbs = options.kma_dbs + kma_dbs
options.kma_dbs = set(options.kma_dbs)
else:
options.kma_dbs = kma_dbs
#########
if Debug:
print(colored("DEBUG: options.kma_dbs", 'yellow'))
print(options.kma_dbs)
## Get databases
for db in options.kma_dbs:
print(colored("\n+ " + db, 'yellow'))
db_folder = HCGB_files.create_subfolder(db, kma_database)
species_identification_KMA.download_kma_database(db_folder, db, Debug)
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###########
## BUSCO ##
###########
if (options.BUSCO_dbs):
print()
HCGB_aes.print_sepLine("*", 50, False)
print("--------- Check BUSCO datasets provided ---------")
BUSCO_folder = HCGB_files.create_subfolder("BUSCO", options.path)
#########
if Debug:
print(colored("DEBUG: options.BUSCO_dbs", 'yellow'))
print(options.BUSCO_dbs)
print("+ BUSCO datasets would be downloaded when executed...")
#BUSCO_caller.BUSCO_retrieve_sets(options.BUSCO_dbs, BUSCO_folder)
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
print("\n*************** Finish *******************\n")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Database module.\n")
return ()
def sketch_database(dict_files, folder, Debug, ksize_n, num_sketch):
"""Sketch sequence files
This function generates a sourmash index, also called sketch, of the sequences
provided in the folder specified.
For speed reasons, we set force=True in add_sequence step to skip over k-mers containing
characters other than ACTG, rather than raising an exception.
:param dict_files: keys are the names of the files and values are the path to the fasta file
:param folder:
:param Debug: True/False to print developing messages.
:param ksize_n: Kmer size value.
:param num_sketch: Number of sketches to include in the hash signature.
:type dict_files: Dictionary
:type folder: string
:type Debug: bool
:type ksize_n: integer
:type num_sketch: integet
:returns: List of SourmashSignature signatures (siglist) and absolute path files generated (siglist_file).
.. attention:: The code to implement this API function was taken and adapted from:
- https://sourmash.readthedocs.io/en/latest/api-example.html
- https://github.com/dib-lab/sourmash/blob/master/sourmash/commands.py
.. seealso:: This function depends on sourmash python module (https://sourmash.readthedocs.io/en/latest/). Some functions employed are:
- :func:`sourmash.MinHash`
- :func:`sourmash.SourmashSignature`
- :func:`sourmash.MinHash.add_sequence`
.. include:: ../../links.inc
"""
### Default: set as option
## num_sketch=5000
## ksize_n=31
minhashes = {}
for name,g in dict_files.items():
print ('\t+ Skecthing sample: ', name)
E = sourmash.MinHash(n=num_sketch, ksize=ksize_n) ## generate hash according to number of sketches and kmer size
for record in screed.open(g):
E.add_sequence(record.sequence, True)
## in add_sequence and for speed reasons, we set force=True to skip over k-mers containing characters other than ACTG, rather than raising an exception.
minhashes[name]= E
## Debug messages
if Debug:
print (colored("\n*** DEBUG: minhashes *****\n", 'red'))
print (type(minhashes))
print (minhashes)
siglist = []
siglist_file = []
### save as signature
HCGB_files.create_folder(folder)
for names,hashes in minhashes.items():
sig1 = SourmashSignature(hashes, name=names)
outfile_name = folder + '/' + str(names) + '.sig'
with open(outfile_name, 'wt') as fp:
save_signatures([sig1], fp)
siglist_file.append(outfile_name)
siglist.append(sig1)
return(siglist_file, siglist)
def run(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
help_info.help_fastq_format()
exit()
elif (options.help_trimm_adapters):
## help on trimm adapters
trimmomatic_call.print_help_adapters()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Trimming samples")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq", ("fastq", "fq", "fastq.gz", "fq.gz"),
options.debug)
## debug message
if (Debug):
HCGB_aes.debug_message("pd_samples_retrieved", 'yellow')
HCGB_main.print_all_pandaDF(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for samples
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "trimm",
options.debug)
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
print("+ Trimming adapters for each sample retrieved...")
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
# Trimming adapters
if (options.adapters):
# Adapter file provided
options.adapters = os.path.abspath(options.adapters)
print("\t- Adapters file provided...")
else:
# Get default adpaters file
print("\t- Default Trimmomatic adapters (v0.39) will be used...")
options.adapters = data_files.data_list(
"available_Trimmomatic_adapters")
## send for each sample
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
commandsSent = {
executor.submit(trimmo_caller, sorted(cluster["sample"].tolist()),
outdir_dict[name], name, threads_job, Debug,
options.adapters): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
print("\n\n+ Trimming samples has finished...")
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## get files generated and generate symbolic link
if not options.project:
dir_symlinks = HCGB_files.create_subfolder('link_files', outdir)
files2symbolic = []
folders = os.listdir(outdir)
## debug message
if (Debug):
print(
colored(
"**DEBUG: generate symbolic links for each file in " +
dir_symlinks + "**", 'yellow'))
for fold in folders:
if fold.endswith(".log"):
continue
else:
this_folder = outdir + '/' + fold
subfiles = os.listdir(this_folder)
for files in subfiles:
files_search = re.search(
r".*trim_R\d{1}.*",
files) ## only paired-end. Todo: single end
if files_search:
files2symbolic.append(this_folder + '/' + files)
HCGB_files.get_symbolic_link(files2symbolic, dir_symlinks)
if (options.skip_report):
print("+ No report generation...")
else:
print("\n+ Generating a report using MultiQC module.")
outdir_report = HCGB_files.create_subfolder("report", outdir)
## call multiQC report module
givenList = [v for v in outdir_dict.values()]
my_outdir_list = set(givenList)
## debug message
if (Debug):
HCGB_aes.debug_message("my_outdir_list for multiqc report",
"yellow")
print(my_outdir_list)
print("\n")
trimm_report = HCGB_files.create_subfolder("trimm", outdir_report)
multiQC_report.multiQC_module_call(my_outdir_list, "Trimmomatic",
trimm_report, "")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% trimm_report)
## create fastqc for trimmed reads
pd_samples_retrieved_trimmed = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
qc.fastqc(pd_samples_retrieved_trimmed, outdir, options,
start_time_partial, "trimmed", Debug)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("\n+ Exiting trimm module.")
return ()
def run_report(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_spaTyper):
## help_format option
get_spa_typing.help_spaTyper()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
## set default
options.batch = False
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
## message header
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Report generation module")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## call assemble using spades
start_time_partial = start_time_total
## absolute path for in & out
options.database = os.path.abspath(options.database)
global input_dir
input_dir = os.path.abspath(options.input)
outdir = ""
## set mode: project/detached
global Project
if (options.detached):
options.project = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
##
print("\n+ Get project information:")
## get files: trimm, assembly, annotation
pd_samples_retrieved = database_user.get_userData_files(options, input_dir)
pd_samples_retrieved['new_name'] = pd_samples_retrieved['name']
## get info: profile, ident, cluster, MGE
pd_samples_info = database_user.get_userData_info(options, input_dir)
## get databases to list
#retrieve_databases = get_options_db(options)
## create output files
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "report",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
print(colored("**DEBUG: pd_samples_info **", 'yellow'))
print(pd_samples_info)
## generate output folder, if necessary
print(
"\n\n\n+ Generate a report summarizing analysis and sample information"
)
if not options.project:
HCGB_files.create_folder(outdir)
outdir_report = outdir
else:
### report generation
outdir_report = HCGB_files.create_subfolder("report", outdir)
## create report with all data
summary_report = HCGB_files.create_subfolder("summary_report",
outdir_report)
print("Folder: ", summary_report)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
########################################
## create species specific report if any
########################################
if (options.species_report):
## Saureus
if options.species_report == "Saureus":
Saureus_specific(pd_samples_retrieved, pd_samples_info, options,
summary_report, outdir_dict)
## else
## to add accordingly
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###########################################################
## create gene fasta sequences retrieval if desired
###########################################################
if options.genes_ids_fasta:
## given a list of genes ids, retrieve sequence for all samples from profile
if os.path.isfile(os.path.abspath(options.genes_ids_fasta)):
in_file = os.path.abspath(options.genes_ids_fasta)
gene_names = [line.rstrip('\n') for line in open(in_file)]
print(
'+ Retrieve selected genes sequences from the profile analysis for each sample.'
)
print('+ Searching gene:')
## get profiles available
results_geneIDs = pd.DataFrame(columns=('sample', 'gene', 'id',
'sequence'))
sample_frame = pd_samples_info.groupby(["name"])
for g in gene_names:
print("\t+", g)
for name, cluster_df in sample_frame:
my_list_profiles = cluster_df.loc[
cluster_df['tag'] == 'profile']['ext'].to_list()
if options.debug:
print("name: ", name)
print("my_list_profiles:")
print(my_list_profiles)
for p in my_list_profiles:
main_profile_folder = cluster_df.loc[
cluster_df['ext'] == p]['dirname'].to_list()[0]
p = p.lower()
if p == 'vfdb':
p = p + '_full'
profile_folder = os.path.join(main_profile_folder, p)
(seq_id, seq_sequence
) = retrieve_genes.retrieve_genes_ids_sequences(
profile_folder, g, Debug)
if (seq_id):
## save results
results_geneIDs.loc[len(results_geneIDs)] = (
name, g, seq_id, seq_sequence)
## save for each gene in a separate fasta file
list_of_genes = set(results_geneIDs['gene'].to_list())
## debug
if Debug:
print("** DEBUG **")
print(results_geneIDs)
print(list_of_genes)
## Save results
genes_folder = HCGB_files.create_subfolder('genes', summary_report)
for gene_retrieved in list_of_genes:
this_frame = results_geneIDs[results_geneIDs['gene'] ==
gene_retrieved]
gene_retrieved_file = os.path.join(genes_folder, gene_retrieved)
gene_retrieved_fasta = gene_retrieved_file + ".fasta"
gene_retrieved_info = gene_retrieved_file + "_info.txt"
fasta_hd = open(gene_retrieved_fasta, 'w')
info_hd = open(gene_retrieved_info, 'w')
for item, row in this_frame.iterrows():
string2write = ">" + row['sample'] + '_' + row[
'gene'] + '\n' + row['sequence'] + '\n'
string2write_info = row['sample'] + '\t' + row[
'gene'] + '\t' + row['id'] + '\n'
fasta_hd.write(string2write)
info_hd.write(string2write_info)
fasta_hd.close()
info_hd.close()
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
########################################
## create gene promoter fasta sequences retrieval if desired
########################################
if options.promoter_bp:
## retrieve as many bp as necessary from genes_ids_fasta
print("** THIS OPTION IS NOT IMPLEMENTED YET... **")
#get_promoter.get_promoter(file, geneOfInterest, basePairs, sampleName, option, debug=False):
########################################
## create gene specific report if any
########################################
if options.genes_ids_profile:
if options.species_report == "Saureus":
if Debug:
print("** options.genes_ids_profile **")
print("Analysis already done for Saureus")
else:
in_file = os.path.abspath(options.genes_ids_profile)
gene_names = [line.rstrip('\n') for line in open(in_file)]
results_Profiles = retrieve_genes.get_genes_profile(
pd_samples_info, gene_names, options.debug, "name")
if options.debug:
print("results_Profiles")
print(results_Profiles)
## open excel writer
name_excel = summary_report + '/gene_ids_profile.xlsx'
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
results_Profiles.to_excel(writer, sheet_name="gene_ids")
## close
writer.save()
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
###############################################
## Search for any additional fasta sequence
###############################################
if options.genes_fasta:
## given a list of fasta sequences search using blast against proteins annotated or genome
print("** THIS OPTION IS NOT IMPLEMENTED YET... **")
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Report generation module.")
return ()
def run_annotation(options):
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_BUSCO):
## information for BUSCO
BUSCO_caller.print_help_BUSCO()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_multiqc):
## information for Multiqc
multiQC_report.multiqc_help()
elif (options.help_Prokka):
## information for Prokka
annotation.print_list_prokka()
exit()
## set default
options.batch = False
###
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Assembly annotation")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
### symbolic links
print("+ Retrieve all genomes assembled...")
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for samples
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "annot",
options.debug)
## annotate
print("+ Annotate assemblies using prokka:")
print("\t-Option: kingdom = ", options.kingdom, "; Annotation mode")
if options.genera == 'Other':
print(
"\t-Option: genera = Off; No genus-specific BLAST databases option provided"
)
else:
print("\t-Option: genera = ", options.genera,
"; Genus-specific BLAST databases option provided")
print("\t-Option: addgenes; Add 'gene' features for each 'CDS' feature")
print("\t-Option: addmrna; Add 'mRNA' features for each 'CDS' feature")
print("\t-Option: cdsrnaolap; Allow [tr]RNA to overlap CDS")
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
## send for each sample
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
commandsSent = {
executor.submit(annot_caller, row['sample'],
outdir_dict[row['name']], options, row['name'],
threads_job): index
for index, row in pd_samples_retrieved.iterrows()
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## get folders
givenList = [v for v in outdir_dict.values()]
protein_files = []
print(
"+ Detail information for each sample could be identified in separate folders:"
)
for folder in givenList:
print('\t + ', folder)
protein_files.extend(
HCGB_main.retrieve_matching_files(folder, '.faa', Debug))
### report generation
if (options.skip_report):
print("+ No annotation report generation...")
else:
### report generation
HCGB_aes.boxymcboxface("Annotation report")
outdir_report = HCGB_files.create_subfolder("report", outdir)
PROKKA_report = HCGB_files.create_subfolder("annotation",
outdir_report)
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% PROKKA_report)
## check if previously report generated
filename_stamp = PROKKA_report + '/.success'
done = 0
if os.path.isdir(PROKKA_report):
if os.path.isfile(filename_stamp):
stamp = HCGB_time.read_time_stamp(filename_stamp)
print(
colored(
"\tA previous report generated results on: %s" % stamp,
'yellow'))
done = 1
## generate report
if done == 0:
## get subdirs generated and call multiQC report module
multiQC_report.multiQC_module_call(givenList, "Prokka",
PROKKA_report, "-dd 2")
print(
'\n+ A summary HTML report of each sample is generated in folder: %s'
% PROKKA_report)
## success stamps
filename_stamp = PROKKA_report + '/.success'
stamp = HCGB_time.print_time_stamp(filename_stamp)
## time stamp
start_time_partial_BUSCO = HCGB_time.timestamp(start_time_total)
## Check each annotation using BUSCO
results = qc.BUSCO_check(input_dir, outdir, options,
start_time_partial_BUSCO, "proteins")
## print to file: results
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Annotation module.")
return ()
def NCBI_DB(strains2get, data_folder, Debug):
"""Donwloads given taxa from NCBI if not available and updates database information.
This function checks in the given folder if strain of interest is available. If not it would connect to NCBI using python module ncbi_genome_download and downloads some information.
:param strains2get: dataframe containing genus, species and NCBI assembly columns among others. See example below.
:param data_folder: Absolute path to database NCBI folder.
:param Debug: Print messages for debugging purposes if desired.
:type strains2get: dataframe
:type data_folder: string
:type Debug: bool
:return: Dataframe of genbank database updated for all available entries.
Columns for the dataframe :file:`strains2get` consist of:
sample,genus,species,strain,BioSample,genome,Plasmids
See and example in file: :file:`/devel/results/strains2get_NCBI_DB.csv` and shown here:
.. include:: ../../devel/results/strains2get_NCBI_DB.csv
:literal:
See example of the return dataframe, containing database information updated in file: :file:`/devel/results/genbank_database.csv` here:
.. include:: ../../devel/results/genbank_database.csv
:literal:
.. seealso:: This function depends on other BacterialTyper functions called:
- :func:`HCGB.functions.file_funtcions.create_folder`
- :func:`HCGB.functions.main_functions.get_data`
- :func:`BacterialTyper.scripts.database_generator.get_dbs`
- :func:`BacterialTyper.scripts.database_generator.get_database`
- :func:`BacterialTyper.scripts.database_generator.NCBIdownload`
- :func:`BacterialTyper.scripts.database_generator.update_db_data_file`
.. include:: ../../links.inc
"""
## set index
strains2get = strains2get.set_index(
'NCBI_assembly_ID', drop=False) ## set new index but keep column
strains2get.index.names = ['ID'] ## rename index
strains2get = strains2get.drop_duplicates()
#########
if Debug:
print(colored("DEBUG: NCBI data provided: ", 'yellow'))
print(strains2get)
## get data existing database
print("+ Create the database in folder: \n", data_folder)
HCGB_files.create_folder(data_folder)
## read database
db_frame = getdbs('NCBI', data_folder, 'genbank', Debug)
database_df = get_database(db_frame, Debug)
#########
if Debug:
print(colored("DEBUG: NCBI genbank database retrieved: ", 'yellow'))
print("db_frame")
print(db_frame)
print()
print("database_df")
print(database_df)
## loop and download
for index, row in strains2get.iterrows():
HCGB_aes.print_sepLine("+", 75, False)
acc_ID = index #strains2get.loc[index]['NCBI_assembly_ID']
info = "Genus: " + strains2get.loc[index][
'genus'] + '\n' + "Species: " + strains2get.loc[index][
'species'] + '\n' + "Strain: " + strains2get.loc[index][
'name'] + '\n' + "ID accession: " + acc_ID + '\n'
dir_path = data_folder + '/genbank/bacteria/' + acc_ID ## module ngd requires to download data in bacteria subfolder under genbank folder
## check if already exists
if acc_ID in database_df.index:
print("\n+ Data is already available in database for: ")
print(colored(info, 'green'))
else:
## download
print("\n+ Downloading data for:")
print(colored(info, 'green'))
data_accID = NCBIdownload(acc_ID, strains2get, data_folder)
this_db = HCGB_main.get_data(data_accID, ',', 'index_col=0')
this_db = this_db.set_index('ID')
database_df = database_df.append(this_db)
## Generate/Update database
database_csv = data_folder + '/genbank_database.csv'
db_updated = update_db_data_file(database_df, database_csv)
print("+ Database has been generated in file: ", database_csv)
return (db_updated)
def download_kma_database(folder, database, debug):
"""
Downloads databases from KMA website.
Using the latest available ftp datasets, this function downloads available datasets using
function :func:`BacterialTyper.scripts.functions.wget_download`.
Ftp site: "ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/latest/"
It also downloads the md5sum for the dataset selected and compares with the
:param folder: Absolute path to folder that contains database.
:param database: Possible options: [bacteria, archaea, protozoa, fungi, plasmids, typestrains, viral].
:param debug: True/false for printing debugging messages.
:type folder: string
:type database: string
:type debug: boolean
.. seealso:: This function depends on other ``BacterialTyper`` functions called:
- :func:`BacterialTyper.scripts.functions.wget_download`
- :func:`BacterialTyper.scripts.functions.check_md5sum`
- :func:`BacterialTyper.scripts.functions.extract`
- :func:`BacterialTyper.scripts.functions.print_time_stamp`
- :func:`BacterialTyper.scripts.functions.read_time_stamp`
- :func:`BacterialTyper.scripts.species_identification_KMA.check_db_indexed`
"""
## ToDo: update with latest version
ftp_site = "http://www.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/latest/"
## In v20190107 there was a plasmid database.
#ftp_site = "ftp://ftp.cbs.dtu.dk/public/CGE/databases/KmerFinder/version/20190107/"
############################################################################
## ToDo: Set automatic: download config file and look for prefix for each
## sample and generate a dictionary to code the prefix for each db.
############################################################################
# Database configuration file - Describes the content of the database
# Each db consist of 5 files with the following extensions: b, comp.b, length.b, seq.b, name
# Other important files are: .name, .kma.entries.all, .kma.entries.deleted, .kma.entries.added, .md5
# db_prefix name description
#bacteria.ATG Bacteria Organisms Bacteria organisms library prefix=ATG
#plasmids.T Bacteria Plasmids Bacteria plasmids library prefix=T
#typestrains.ATG Bacteria Type Strains Bacteria type strains library prefix=ATG
#fungi.ATG Fungi Fungi library prefix=ATG
#protozoa.ATG Protozoa Protozoa library prefix=ATG
#archaea.ATG Archaea Archaea library prefix=ATG
HCGB_files.create_folder(folder)
## debug message
if (debug):
print (colored("Function call: download_kma_database " + folder + ' ' + database + '\n','yellow'))
## prefix
if (database == 'plasmids'):
prefix = '.T'
elif (database == 'viral'):
prefix = '.TG'
else:
prefix = '.ATG'
index_name = os.path.join(folder, database + prefix)
## check if already download
return_code_down = False
if os.path.exists(folder):
return_code_down = check_db_indexed(index_name, folder)
## debug message
if (debug):
print (colored("Folder database is already available:" + folder,'yellow'))
if (return_code_down == False): ## folder does not exists
## Download data
print ("\t+ Downloading data now, it may take a while....")
## debug message
if (debug):
print (colored("Download files via function wget_download:",'yellow'))
## connect to url
url = ftp_site + database + '.tar.gz'
HCGB_sys.wget_download(url, folder)
md5_url = ftp_site + database + '.md5'
HCGB_sys.wget_download(md5_url, folder)
print ("\n\t+ Data downloaded.....")
## get files
files = os.listdir(folder)
md5_sum = ""
for f in files:
if f.endswith('tar.gz'):
tar_file = folder + '/' + f
elif f.endswith('md5'):
md5_sum = folder + '/' + f
## check md5sum
print ("\t+ Checking for integrity using md5sum")
# get md5 sum from source
md5_string = ""
with open(md5_sum, 'r') as myfile:
line = myfile.read()
line = re.sub(r"\s", ',', line)
md5_string = line.split(",")[0]
## calculate md5 for file
result_md5 = HCGB_sys.check_md5sum(md5_string, tar_file) ## FIXME: Not conda supported
if (result_md5 == True):
## debug message
if (debug):
print (colored("result md5sum matches code provided for file " + tar_file,'yellow'))
# extract
print ("\t+ Extracting database into destination folder: " + folder)
HCGB_files.extract(tar_file, folder)
else:
print (colored("*** ERROR: Some error occurred during the downloading and file is corrupted ***", 'red'))
return ("Error")
## database should be unzipped and containing files...
return_code_extract = check_db_indexed(index_name, folder)
if (return_code_extract):
print("+ Database (%s) successfully extracted in folder: %s..." %(database, folder))
else:
string = "*** ERROR: Some error occurred during the extraction of the database (%s). Please check folder (%s) and downloading and file is corrupted ***" %(database, folder)
print (colored(string, 'red'))
return ("Error")
## print timestamp
filename_stamp = folder + '/.success'
stamp = HCGB_time.print_time_stamp(filename_stamp)
def mapReads(option, reads, folder, name, STAR_exe, genomeDir, limitRAM_option,
num_threads, Debug):
"""
Map reads using STAR software. Some parameters are set for small RNA Seq.
Parameters set according to ENCODE Project directives for small RNAs
https://www.encodeproject.org/rna-seq/small-rnas/
:param option: If multiple files to map, use loaded genome (LoadAndKeep) if only one map, anything else.
:param reads: List containing absolute path to reads (SE or PE)
:param folder: Path for output results
:param name: Sample name
:param STAR_exe: Executable path for STAR binary
:param genomeDir:
:param limitRAM_option: maximum available RAM (bytes) for map reads process. Default: 40000000000
:param num_threads:
:type option: string
:type reads: list
:type folder: string
:type name: string
:type STAR_exe: string
:type genomeDir: string
:type limitRAM_option: int
:type num_threads: int
"""
## open file
print("\t+ Mapping sample %s using STAR" % name)
if not os.path.isdir(folder):
folder = files_functions.create_folder(folder)
##
bam_file_name = os.path.join(folder, 'Aligned.sortedByCoord.out.bam')
## read is a list with 1 or 2 read fastq files
jread = " ".join(reads)
## prepare command
cmd = "%s --genomeDir %s --runThreadN %s " % (STAR_exe, genomeDir,
num_threads)
cmd = cmd + "--limitBAMsortRAM %s --outFileNamePrefix %s " % (
limitRAM_option, folder + '/')
## some common options
cmd = cmd + "--alignSJDBoverhangMin 1000 --outFilterMultimapNmax 1 --outFilterMismatchNoverLmax 0.03 "
cmd = cmd + "--outFilterScoreMinOverLread 0 --outFilterMatchNminOverLread 0 --outFilterMatchNmin 16 "
cmd = cmd + "--alignIntronMax 1 --outSAMheaderHD @HD VN:1.4 SO:coordinate --outSAMtype BAM SortedByCoordinate "
## Multiple samples or just one?
if option == 'LoadAndKeep':
cmd = cmd + "--genomeLoad LoadAndKeep"
else:
cmd = cmd + "--genomeLoad NoSharedMemory"
## ReadFiles
cmd = cmd + " --readFilesIn %s " % jread
## logfile & errfile
logfile = os.path.join(folder, 'STAR.log')
errfile = os.path.join(folder, 'STAR.err')
cmd = cmd + ' > ' + logfile + ' 2> ' + errfile
## sent command
mapping_code = system_call_functions.system_call(cmd, False, True)
return (mapping_code)
def parse_options(arg_dict):
outdir = os.path.abspath(arg_dict.output_folder)
## TODO: Now set as mutually_exclusive group. It might be Set to multiple options
## ATTENTION: df_accID merge generated dataframe
## --------------------------------------- ##
## GFF or GBF file
## --------------------------------------- ##
if (arg_dict.annot_file):
arg_dict.annot_file = os.path.abspath(arg_dict.annot_file)
# *************************** ##
## multiple files provided
# *************************** ##
if (arg_dict.batch):
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Multiple annotation file provided option:',
'yellow')
debug_message('arg_dict.annot_file: ' + arg_dict.annot_file,
'yellow')
## check if ok
BacDup_functions.file_readable_check(arg_dict.annot_file)
print(
colored('\t* Multiple annotation files provided .......[OK]',
'green'))
dict_entries = HCGB_main.file2dictionary(arg_dict.annot_file, ',')
## debug messages
if (arg_dict.debug):
debug_message('dict_entries: ', 'yellow')
debug_message(dict_entries, 'yellow')
debug_message('+++++++++++++++++++++++++++++++\n\n')
# *************************** ##
## single file provided
# *************************** ##
else:
dict_entries = {}
print(colored('\t* Annotation file:.......[OK]', 'green'))
if (arg_dict.sample_name):
sample_name = arg_dict.sample_name
else:
sample_name = "sample"
##
dict_entries[sample_name] = arg_dict.annot_file
## create dataframe df_accID to match other formats
df_accID = pd.DataFrame(
columns=(BacDup_functions.columns_accID_table()))
for name, file_annot in dict_entries.items():
file_annot = os.path.abspath(file_annot)
## init all
genome = ""
prot = ""
gff = ""
gbk = ""
plasmid_count = ""
plasmid_id = ""
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message(
'dict_entries check annotation files provided option:',
'yellow')
debug_message('name: ' + name, 'yellow')
debug_message('file_annot: ' + file_annot, 'yellow')
## check file is valid
BacDup_functions.file_readable_check(file_annot)
## get format
format = format_checker.is_format(file_annot, arg_dict.debug)
if (arg_dict.debug):
debug_message('format: ' + format, 'yellow')
## parse accordingly
taxonomy = ""
organism = ""
taxonomy_string = ""
genus = ""
if (format == 'gbk'):
## get information from each sample
(taxonomy,
organism) = BacDup.scripts.functions.get_gbk_information(
file_annot, arg_dict.debug)
## plasmid_count, plasmid_id not available
elif (format == 'gff'):
if (arg_dict.ref_file):
arg_dict.ref_file = os.path.abspath(arg_dict.ref_file)
BacDup_functions.file_readable_check(arg_dict.ref_file)
if (arg_dict.batch):
ref_entries = HCGB_main.file2dictionary(
arg_dict.ref_file, ',')
genome = ref_entries[name]
else:
genome = arg_dict.ref_file
## save into dataframe
if len(taxonomy) > 1:
genus = taxonomy[-1]
taxonomy_string = ";".join(taxonomy)
dir_path = os.path.abspath(os.path.dirname(file_annot))
df_accID.loc[len(df_accID)] = (name, dir_path, genus, organism,
taxonomy_string, genome, file_annot,
format, prot, plasmid_count,
";".join(plasmid_id))
## --------------------------------------- ##
## NCBI RefSeq/Genbank IDs: GCA_XXXXXXXX.1; GCF_XXXXXXXXX.1
## --------------------------------------- ##
elif (arg_dict.GenBank_id):
## get database path
if (arg_dict.db_folder):
db_folder = HCGB_files.create_folder(
os.path.abspath(arg_dict.db_folder))
else:
db_folder = HCGB_files.create_subfolder(
"db", os.path.abspath(arg_dict.output_folder))
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('GenBank ID option:', 'yellow')
debug_message('db_folder: ' + db_folder, 'yellow')
# *************************** ##
## batch file
# *************************** ##
if (arg_dict.batch):
arg_dict.GenBank_id = os.path.abspath(arg_dict.GenBank_id)
## debug messages
if (arg_dict.debug):
debug_message('GenBank ID batch file provided:', 'yellow')
debug_message('arg_dict.GenBank_id: ' + arg_dict.GenBank_id,
'yellow')
## check is a file and readable
BacDup_functions.file_readable_check(arg_dict.GenBank_id)
print(
colored('\t* Multiple NCBI GenBank IDs in a file .......[OK]',
'green'))
print()
## call IDs into a list and create tmp folder
strains2get = HCGB_main.readList_fromFile(arg_dict.GenBank_id)
strains2get = list(filter(None, strains2get))
## debug messages
if (arg_dict.debug):
debug_message('strains2get: ' + str(strains2get), 'yellow')
## call NCBI_downloader
df_accID = BacDup.scripts.NCBI_downloader.NCBI_download_list(
strains2get, db_folder, arg_dict.debug,
arg_dict.assembly_level)
# *************************** ##
## single GenBank ID
# *************************** ##
else:
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Single NCBI GenBank IDs provided option:',
'yellow')
debug_message('arg_dict.GenBank_id: ' + arg_dict.GenBank_id,
'yellow')
debug_message('db_folder: ' + db_folder, 'yellow')
debug_message('+++++++++++++++++++++++++++++++')
## download
print(colored('\t* A NCBI GenBank ID:.......[OK]', 'green'))
print()
HCGB_aes.print_sepLine("+", 75, False)
df_accID = BacDup.scripts.NCBI_downloader.NCBIdownload(
arg_dict.GenBank_id, db_folder, arg_dict.debug)
## --------------------------------------- ##
## NCBI Taxonomy ID:
## --------------------------------------- ##
elif (arg_dict.tax_id):
#################
## get tax ids
#################
if (arg_dict.batch):
print(
colored('\t* Multiple NCBI Taxonomy IDs in a file .......[OK]',
'green'))
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Multiple NCBI Taxonomy IDs provided option:',
'yellow')
## check is a file and readable
BacDup_functions.file_readable_check(arg_dict.tax_id)
## get IDs into a list
taxIDs2get = HCGB_main.readList_fromFile(arg_dict.tax_id)
else:
print(colored('\t* A NCBI Taxonomy ID:.......[OK]', 'green'))
taxIDs2get = [arg_dict.tax_id]
print()
##################################
## init ete NCBI taxonomy database
##################################
print('+ Initiate NCBI taxonomy database...')
ncbi = taxonomy_retrieval.init_db_object(arg_dict.debug)
string_info_total = []
for taxid in taxIDs2get:
## parse
info = taxonomy_retrieval.parse_taxid(taxid, ncbi, 'unravel',
arg_dict.debug)
print()
## debug messages
if arg_dict.debug:
debug_message(
"+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
debug_message('info\n', "yellow")
print(info)
## append if more
string_info_total.extend(info)
## convert to list of strings
string_info_total = [str(int) for int in string_info_total]
## assume all belong to same superkingdom if children of same tax_id
group_obtained = taxonomy_retrieval.get_superKingdom(
string_info_total[0], ncbi, arg_dict.debug)
#################
## get database path
#################
if (arg_dict.db_folder):
db_folder = HCGB_files.create_folder(
os.path.abspath(arg_dict.db_folder))
else:
db_folder = HCGB_files.create_subfolder("db", outdir)
## debug messages
if arg_dict.debug:
debug_message(
"+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
debug_message('group_obtained: ' + group_obtained, "yellow")
debug_message('db_folder: ' + db_folder, "yellow")
debug_message(
'arg_dict.assembly_level: ' + arg_dict.assembly_level,
"yellow")
debug_message('arg_dict.section: ' + arg_dict.section, "yellow")
##################################
## get GenBank entries selected
##################################
(strains2get,
allstrains_available) = taxonomy_retrieval.get_GenBank_ids(
db_folder,
string_info_total,
int(arg_dict.k_random),
arg_dict.debug,
assembly_level_given=arg_dict.assembly_level,
group_given=group_obtained,
section_given=arg_dict.section)
## print list and dictionary of possible and selected taxIDs
outdir = os.path.abspath(arg_dict.output_folder)
info_dir = HCGB_files.create_subfolder("info", outdir)
input_info_dir = HCGB_files.create_subfolder("input", info_dir)
HCGB_main.printList2file(
os.path.join(input_info_dir, 'Downloaded.txt'), strains2get)
HCGB_main.printList2file(
os.path.join(input_info_dir, 'all_entries.txt'),
allstrains_available)
## save into file
file_info = os.path.join(input_info_dir, 'info.txt')
## stop here if dry_run
if arg_dict.dry_run:
print()
HCGB_aes.print_sepLine("*", 75, False)
print(
"ATTENTION: Dry run mode selected. Stopping the process here.")
HCGB_aes.print_sepLine("*", 75, False)
print("+ All available entries listed and printed in file:\n\t" +
os.path.join(input_info_dir, 'all_entries.txt'))
print("+ Subset of entries generated and printed in file:\n\t" +
os.path.join(input_info_dir, 'Downloaded.txt'))
print(
"\n\nIf random numbers selected, take into account re-running this process might produce different results.\n"
)
HCGB_aes.print_sepLine("*", 75, False)
print()
exit()
#################
## call NCBI_downloader
#################
df_accID = BacDup.scripts.NCBI_downloader.NCBI_download_list(
strains2get, db_folder, arg_dict.debug, arg_dict.assembly_level)
## --------------------------------------- ##
## Previous BacDup analysis folder
## --------------------------------------- ##
## TODO
elif (arg_dict.project):
print(
colored(
'\t* A previous BacDup analysis project folder:.......[OK]',
'green'))
## create df_accID to store data
## TODO
## Returns dataframe with information
df_accID = df_accID.set_index('new_name')
return (df_accID)
def ARIBA_ident(options, pd_samples_retrieved, outdir_dict, retrieve_databases,
start_time_partial):
HCGB_aes.boxymcboxface("ARIBA Identification")
##################
## check status ##
##################
databases2use = [] ## path, db name
card_trick_info = ""
print('+ Check databases status: ')
for index, db2use in retrieve_databases.iterrows():
## index_name
if (db2use['source'] == 'ARIBA'):
index_status = ariba_caller.check_db_indexed(db2use['path'], 'YES')
if (index_status == True):
#print (colored("\t+ Databases %s seems to be fine...\n\n" % db2use['db'], 'green'))
databases2use.append([db2use['path'], db2use['db']])
## prepare card database ontology for later
if (db2use['db'] == 'card'):
card_trick_info = card_trick_caller.prepare_card_data(
options.database)
## check status of other databases if any
# else:
## debug message
if (Debug):
print(colored("**DEBUG: databases2use\n**", 'yellow'))
print(databases2use)
if (card_trick_info):
print(
colored("**DEBUG: card_trick_info: " + card_trick_info + " **",
'yellow'))
######################################################
## Start identification of samples
######################################################
print("\n+ Send ARIBA identification jobs...")
## get outdir folders
outdir_samples = pd.DataFrame(columns=('sample', 'dirname', 'db',
'output'))
# Group dataframe by sample name
sample_frame = pd_samples_retrieved.groupby(["name"])
for name, cluster in sample_frame:
for db2use in databases2use:
tmp = get_outfile(outdir_dict[name], name, db2use[0])
outdir_samples.loc[len(outdir_samples)] = (name, outdir_dict[name],
db2use[1], tmp)
## multi-index
outdir_samples = outdir_samples.set_index(['sample', 'db'])
## debug message
if (Debug):
print(colored("**DEBUG: outdir_samples **", 'yellow'))
print(outdir_samples)
######################################################
## send for each sample
######################################################
## ariba assembly cutoff
if not (options.ARIBA_cutoff):
options.ARIBA_cutoff = 0.90
## optimize threads
name_list = set(pd_samples_retrieved["name"].tolist())
threads_job = HCGB_main.optimize_threads(
options.threads, len(name_list)) ## threads optimization
max_workers_int = int(options.threads / threads_job)
## debug message
if (Debug):
print(
colored("**DEBUG: options.threads " + str(options.threads) + " **",
'yellow'))
print(
colored("**DEBUG: max_workers " + str(max_workers_int) + " **",
'yellow'))
print(
colored("**DEBUG: cpu_here " + str(threads_job) + " **", 'yellow'))
## loop
results_df = pd.DataFrame()
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers_int) as executor:
for db2use in databases2use:
print(colored("+ Working with database: " + db2use[1], 'yellow'))
## send for each sample
commandsSent = {
executor.submit(
ariba_run_caller,
db2use[0],
db2use[1], ## database path & dbname
sorted(cluster["sample"].tolist()), ## files
outdir_samples.loc[(name, db2use[1]), 'output'], ## output
threads_job,
options.ARIBA_cutoff): name
for name, cluster in sample_frame
}
for cmd2 in concurrent.futures.as_completed(commandsSent):
details = commandsSent[cmd2]
try:
data = cmd2.result()
except Exception as exc:
print('***ERROR:')
print(cmd2)
print('%r generated an exception: %s' % (details, exc))
print("+ Jobs finished for database %s ..." % db2use[1])
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
print()
print(
"+ Collecting information for each sample analyzed for database: "
+ db2use[1])
## check results for each database
results_df_tmp = virulence_resistance.check_results(
db2use[1], outdir_samples, options.ARIBA_cutoff,
card_trick_info)
results_df = pd.concat([results_df, results_df_tmp])
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
######################################################
## Generate final report for all samples
######################################################
## ariba summary results all samples
print(
"\n + Generate a summary file for all samples and one for each database employed..."
)
## parse results
if Project:
final_dir = input_dir + '/report/profile'
HCGB_files.create_folder(final_dir)
else:
final_dir = os.path.abspath(options.output_folder)
##
vfdb = False
subfolder = HCGB_files.create_subfolder("ariba_summary", final_dir)
## subfolder_samples = functions.create_subfolder("samples", final_dir) ## TODO: Copy all xlsx files to a common folder. Is it necessary?
## open excel writer
name_excel = final_dir + '/profile_summary.xlsx'
writer = pd.ExcelWriter(name_excel, engine='xlsxwriter')
for database, data in outdir_samples.groupby(level='db'): ## fix
report_files_databases = {}
for sample, data2 in data.groupby(level='sample'): ## fix
file_report = data2.loc[sample, database]['output'] + '/report.tsv'
if os.path.isfile(file_report): ## check if exists
report_files_databases[sample] = file_report
outfile_summary = subfolder + "/"
if database.endswith('card_prepareref/'):
outfile_summary = outfile_summary + 'CARD_summary'
name_db = 'CARD'
elif database.endswith('vfdb_full_prepareref/'):
outfile_summary = outfile_summary + 'VFDB_summary'
name_db = 'VFDB'
vfdb = True
else:
## TODO: check if there are multiple 'other' databases
## Different databases provided (different to VFDB and CARD) would collapse file
outfile_summary = outfile_summary + 'Other_summary'
name_db = 'other'
## call ariba summary to summarize results
csv_all = ariba_caller.ariba_summary_all(outfile_summary,
report_files_databases)
if not csv_all == 'NaN':
csv2excel = pd.read_csv(csv_all, header=0, sep=',')
## write excel
name_tab = name_db + '_found'
csv2excel.to_excel(writer, sheet_name=name_tab)
## results_df contains excel and csv files for each sample and for each database
list_databases = set(results_df['database'].to_list())
for db in list_databases:
df_db = results_df[results_df['database'] == db]['csv']
dict_samples = df_db.to_dict()
merge_df = pd.DataFrame()
for sample in dict_samples:
if os.path.isfile(dict_samples[sample]):
df = pd.read_csv(dict_samples[sample], header=0, sep=",")
df = df.set_index('Genes')
df2 = df.rename(columns={'Status': sample}, inplace=True)
df2 = df[[sample]]
## add to a common dataframe
merge_df = pd.concat([merge_df, df2], axis=1, sort=True)
merge_df.fillna("NaN", inplace=True)
trans_df = merge_df.transpose()
## write excel
name_tab = db + '_all'
trans_df.to_excel(writer, sheet_name=name_tab)
## close
writer.save()
######################################################
## print additional information for VFDB
######################################################
if (vfdb):
print("\n\n")
HCGB_aes.print_sepLine("*", 50, False)
print("+ Check VFDB details in files downloaded from vfdb website:")
files_VFDB = virulence_resistance.check_VFDB(final_dir +
'/VFDB_information')
HCGB_aes.print_sepLine("*", 50, False)
######################################################
print("\n+ Please check additional summary files generated at folder ",
final_dir)
print("+ Go to website: https://jameshadfield.github.io/phandango/#/")
print(
"+ For each database upload files *phandango.csv and *phandango.tre and visualize results"
)
def run_input(arg_dict):
"""Main function of the input_parser module in BacDup package.
This module prepares data for later gene duplication analysis.
It allows the user to provide either a single sample, multiple samples, NCBI
GenBank IDs or NCBI taxonomy IDs to retrieve and obtain the annotation data.
"""
## help message
if (arg_dict.input_help):
help_input()
exit()
BacDup_functions.pipeline_header('BacDup')
HCGB_aes.boxymcboxface("Preparing input files")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## init time
start_time_total = time.time()
## absolute path for in & out
#input_dir = os.path.abspath(options.input)
outdir = os.path.abspath(arg_dict.output_folder)
## output folder
print("\n+ Create output folder(s):")
HCGB_files.create_folder(outdir)
## set defaults
if not (arg_dict.assembly_level):
arg_dict.assembly_level = 'complete'
if not (arg_dict.section):
arg_dict.section = 'genbank'
## project or detached?
if arg_dict.detached:
arg_dict.project = False
final_dir = outdir
data_dir = outdir
else:
arg_dict.project = True
print(
"+ Generate a directory containing information within the project folder provided"
)
final_dir = HCGB_files.create_subfolder("info", outdir)
## debug messages
if (arg_dict.debug):
debug_message('+++++++++++++++++++++++++++++++')
debug_message('Project/Detached option:', 'yellow')
debug_message('arg_dict.detached: ' + str(arg_dict.detached), 'yellow')
debug_message('arg_dict.project: ' + str(arg_dict.project), 'yellow')
debug_message('outdir:' + outdir, 'yellow')
debug_message('final_dir:' + final_dir, 'yellow')
debug_message('+++++++++++++++++++++++++++++++')
## get files
print()
HCGB_aes.print_sepLine("-", 50, False)
print('+ Getting input information provided... ')
print('+ Several options available:')
print('\t* Single/Multiple Annotation file:')
print('\t |-- GenBank format files')
print('\t |-- GFF files + Reference fasta files required')
print('\n\t* Single/Multiple NCBI GenBank IDs')
print('\n\t* Single/Multiple NCBI taxonomy IDs + Options')
print('\n\t* A previous BacDup project folder')
print('\n+ Check the option provided...')
time.sleep(1)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
#################################################
## Parse and obtain the type of input information provided
#################################################
df_accID = parse_options(arg_dict)
## pd.DataFrame: 'new_name','folder','genus',
## 'species','taxonomy','genome',
## 'annot_file','format_annot_file', 'proteins',
## 'plasmids_number','plasmids_ID'))
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## parse information accordingly
parse_information(arg_dict, df_accID, outdir)
### report generation
HCGB_aes.boxymcboxface("Summarizing input files")
outdir_report = HCGB_files.create_subfolder("report", outdir)
input_report = HCGB_files.create_subfolder("input", outdir_report)
## add df_accID.loc[sample,] information as csv into input folder
df_accID.to_csv(os.path.join(input_report, 'info.csv'),
index=True,
header=True)
## maybe add a summary of the files?
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting Input module.")
return ()
def run_cluster(options):
## init time
start_time_total = time.time()
##################################
### show help messages if desired
##################################
if (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_Mash):
## information for Min Hash Software
min_hash_caller.helpMash()
exit()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Clustering samples")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
project_mode = True
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
else:
options.project = True
outdir = input_dir
## get files
if options.reads:
if options.noTrim:
## raw reads
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "fastq",
("fastq", "fq", "fastq.gz", "fq.gz"), options.debug)
else:
## trimm reads
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## keep only R1 reads if paired-end
if options.pair:
pd_samples_retrieved = pd_samples_retrieved.loc[
pd_samples_retrieved['read_pair'] == "R1"]
else:
## default
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "assembly", ["fna"], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
# exit if empty
if pd_samples_retrieved.empty:
print(
"No data has been retrieved from the project folder provided. Exiting now..."
)
exit()
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "mash",
options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: outdir_dict **", 'yellow'))
print(outdir_dict)
## get databases to check
retrieve_databases = get_options_db(options)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## remove samples if specified
if options.ex_sample:
ex_samples = HCGB_main.get_info_file(options.ex_sample)
retrieve_databases = retrieve_databases.loc[~retrieve_databases.index.
isin(ex_samples)]
## debug message
if (Debug):
print(colored("**DEBUG: retrieve_database **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(retrieve_databases)
## check if all samples in user_data or genbank are indexed
siglist_all = []
for index, row in retrieve_databases.iterrows():
if not row['path'] == 'NaN':
if (Debug):
HCGB_aes.print_sepLine("*", 25, False)
print(row)
if all([
int(options.kmer_size) == int(row['ksize']),
int(options.n_sketch) == int(row['num_sketch'])
]):
siglist_all.append(
min_hash_caller.read_signature(row['path'],
options.kmer_size))
continue
## index assembly or reads...
(sigfile, siglist) = generate_sketch(row['folder'], row['original'],
index, options.kmer_size,
options.n_sketch, Debug)
retrieve_databases.loc[index]['path'] = sigfile
retrieve_databases.loc[index]['ksize'] = options.kmer_size
retrieve_databases.loc[index]['num_sketch'] = options.n_sketch
siglist_all.append(siglist)
### Cluster project samples
print(colored("\n+ Collect project data", 'green'))
print("+ Generate mash sketches for each sample analyzed...")
pd_samples_retrieved = pd_samples_retrieved.set_index('name')
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieved **", 'yellow'))
print(pd_samples_retrieved)
## init dataframe for project data
colname = ["source", "name", "path", "original", "ksize", "num_sketch"]
pd_samples_sketched = pd.DataFrame(columns=colname)
for index, row in pd_samples_retrieved.iterrows():
if index in retrieve_databases.index:
print(
colored(
'\t+ Sketched signature (%s) available within user data...'
% index, 'yellow'))
continue
this_sig = outdir_dict[index] + '/' + index + '.sig'
if os.path.exists(this_sig):
## File signature might exist
## read original
file2print = outdir_dict[index] + '/.original'
if not os.path.exists(file2print):
original = ['NaN']
else:
original = HCGB_main.readList_fromFile(file2print)
if all([
int(options.kmer_size) == int(original[1]),
int(options.n_sketch) == int(original[2])
]):
siglist_all.append(
min_hash_caller.read_signature(this_sig,
options.kmer_size))
pd_samples_sketched.loc[len(pd_samples_sketched)] = (
'project_data', index, this_sig, row['sample'],
options.kmer_size, options.n_sketch)
print(
colored(
'\t+ Sketched signature available (%s) in project folder...'
% index, 'green'))
continue
print(
colored('\t+ Sketched signature to be generated: (%s)...' % index,
'yellow'))
## index assembly or reads...
(sigfile, siglist) = generate_sketch(outdir_dict[index], row['sample'],
index, options.kmer_size,
options.n_sketch, Debug)
pd_samples_sketched.loc[len(pd_samples_sketched)] = ('project_data',
index, sigfile,
row['sample'],
options.kmer_size,
options.n_sketch)
siglist_all.append(siglist)
print("\n+ Clustering sequences...")
pd_samples_sketched = pd_samples_sketched.set_index('name')
####
if retrieve_databases.empty:
cluster_df = pd_samples_sketched
else:
tmp = retrieve_databases[[
'source', 'db', 'path', 'original', 'ksize', 'num_sketch'
]]
tmp = tmp.rename(columns={'db': 'name'})
tmp.set_index('name')
if (Debug):
print(colored("**DEBUG: tmp **", 'yellow'))
print(tmp)
## merge both dataframes
cluster_df = pd.concat([pd_samples_sketched, tmp],
join='inner',
sort=True)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_sketched **", 'yellow'))
print(pd_samples_sketched)
print(colored("**DEBUG: cluster_df **", 'yellow'))
print(cluster_df)
print(colored("**DEBUG: Signatures **", 'yellow'))
print(siglist_all)
print(colored("**DEBUG: length siglist_all **", 'yellow'))
print(len(siglist_all))
## Assign Colors colorLabels
color_df = cluster_df.filter(["source"], axis=1)
color_df["color"] = "r" ## red::genbank
## project data
project_data = list(color_df[color_df["source"] == "project_data"].index)
color_df.loc[color_df.index.isin(project_data),
"color"] = "g" ## green::project_data
## user_data
user_data = list(color_df[color_df["source"] == "user_data"].index)
color_df.loc[color_df.index.isin(user_data),
"color"] = "b" ## blue::user_data
colorLabels = color_df['color'].to_dict()
if Debug:
print(color_df)
print(colorLabels)
## parse results
if options.project:
outdir_report = HCGB_files.create_subfolder("report", outdir)
#final_dir = outdir + '/report/cluster'
final_dir = functions.create_subfolder("cluster", outdir_report)
else:
final_dir = outdir
## compare
name = 'cluster_' + str(HCGB_time.create_human_timestamp())
tag_cluster_info = final_dir + '/' + name
print('+ Saving results in folder: ', final_dir)
print('\tFile name: ', name)
(DataMatrix, labeltext) = min_hash_caller.compare(siglist_all,
tag_cluster_info, Debug)
## get colorLabels
## plot images
pdf = True
cluster_returned = min_hash_caller.plot(DataMatrix, labeltext,
tag_cluster_info, pdf, colorLabels)
## generate newick tree
min_hash_caller.get_Newick_tree(cluster_returned, DataMatrix, labeltext,
tag_cluster_info)
return ()
def run_ident(options):
"""
Main function acting as an entry point to the module *ident*.
Arguments:
.. seealso:: Additional information to PubMLST available datasets.
- :doc:`PubMLST datasets<../../../data/PubMLST_datasets>`
"""
##################################
### show help messages if desired
##################################
if (options.help_format):
## help_format option
sampleParser.help_format()
exit()
elif (options.help_project):
## information for project
help_info.project_help()
exit()
elif (options.help_KMA):
## information for KMA Software
species_identification_KMA.help_kma_database()
exit()
elif (options.help_MLSTar):
## information for KMA Software
MLSTar.help_MLSTar()
exit()
## init time
start_time_total = time.time()
## debugging messages
global Debug
if (options.debug):
Debug = True
else:
Debug = False
### set as default paired_end mode
if (options.single_end):
options.pair = False
else:
options.pair = True
### species_identification_KMA -> most similar taxa
HCGB_aes.pipeline_header("BacterialTyper", ver=pipeline_version)
HCGB_aes.boxymcboxface("Species identification")
print("--------- Starting Process ---------")
HCGB_time.print_time()
## absolute path for in & out
input_dir = os.path.abspath(options.input)
outdir = ""
## Project mode as default
global Project
if (options.detached):
options.project = False
project_mode = False
outdir = os.path.abspath(options.output_folder)
Project = False
else:
options.project = True
outdir = input_dir
Project = True
## get files
pd_samples_retrieved = sampleParser.files.get_files(
options, input_dir, "trim", ['_trim'], options.debug)
## debug message
if (Debug):
print(colored("**DEBUG: pd_samples_retrieve **", 'yellow'))
print(pd_samples_retrieved)
## generate output folder, if necessary
print("\n+ Create output folder(s):")
if not options.project:
HCGB_files.create_folder(outdir)
## for each sample
outdir_dict = HCGB_files.outdir_project(outdir, options.project,
pd_samples_retrieved, "ident",
options.debug)
## let's start the process
print(
"+ Generate an species typification for each sample retrieved using:")
print("(1) Kmer alignment (KMA) software.")
print("(2) Pre-defined databases by KMA or user-defined databases.")
## get databases to check
retrieve_databases = get_options_db(options)
## time stamp
start_time_partial = HCGB_time.timestamp(start_time_total)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve_database **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(retrieve_databases)
######## KMA identification
dataFrame_kma = KMA_ident(options, pd_samples_retrieved, outdir_dict,
retrieve_databases, start_time_partial)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve results to summarize **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("dataframe_kma")
print(dataFrame_kma)
## exit if viral search
skip = False
if (len(options.kma_dbs) == 1):
for i in options.kma_dbs:
if (i == 'viral'):
print()
MLST_results = ''
options.fast = True
skip = True
## what if only plasmids?
## do edirect and MLST if bacteria
if (not skip):
dataFrame_edirect = pd.DataFrame()
######## EDirect identification
#dataFrame_edirect = edirect_ident(dataFrame_kma, outdir_dict, Debug)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(colored("**DEBUG: retrieve results from NCBI **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("dataFrame_edirect")
print(dataFrame_edirect)
######## MLST identification
MLST_results = MLST_ident(options, dataFrame_kma, outdir_dict,
dataFrame_edirect, retrieve_databases)
## functions.timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
## debug message
if (Debug):
print(
colored("**DEBUG: retrieve results to summarize **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print("MLST_results")
print(MLST_results)
## generate summary for sample: all databases
## MLST, plasmids, genome, etc
HCGB_aes.boxymcboxface("Results Summary")
#####################################
## Summary identification results ##
#####################################
## parse results
if options.project:
final_dir = os.path.join(outdir, 'report', 'ident')
HCGB_files.create_folder(final_dir)
else:
final_dir = outdir
###
excel_folder = HCGB_files.create_subfolder("samples", final_dir)
print('+ Print summary results in folder: ', final_dir)
print('+ Print sample results in folder: ', excel_folder)
# Group dataframe results summary by sample name
sample_results_summary = dataFrame_kma.groupby(["Sample"])
## debug message
if (Debug):
print(colored("**DEBUG: sample_results_summary **", 'yellow'))
print(sample_results_summary)
##
results_summary_KMA = pd.DataFrame()
MLST_all = pd.DataFrame()
for name, grouped in sample_results_summary:
## create a excel and txt for sample
name_sample_excel = excel_folder + '/' + name + '_ident.xlsx'
name_sample_csv = outdir_dict[
name] + '/ident_summary.csv' ## check in detached mode
writer_sample = pd.ExcelWriter(
name_sample_excel, engine='xlsxwriter') ## open excel handle
## subset dataframe & print result
results_summary_toPrint_sample = grouped[[
'Sample', '#Template', 'Query_Coverage', 'Template_Coverage',
'Depth', 'Database'
]]
results_summary_toPrint_sample.to_excel(
writer_sample, sheet_name="KMA") ## write excel handle
results_summary_toPrint_sample.to_csv(
name_sample_csv) ## write csv for sample
## read MLST
if MLST_results:
if name in MLST_results:
sample_MLST = pd.read_csv(MLST_results[name],
header=0,
sep=',')
sample_MLST['genus'] = dataFrame_edirect.loc[
dataFrame_edirect['sample'] == name, 'genus'].values[0]
sample_MLST['species'] = dataFrame_edirect.loc[
dataFrame_edirect['sample'] == name, 'species'].values[0]
sample_MLST.to_excel(writer_sample,
sheet_name="MLST") ## write excel handle
## Return information to excel
MLST_all = pd.concat([MLST_all, sample_MLST])
## close excel handle
writer_sample.save()
##
name_excel = final_dir + '/identification_summary.xlsx'
print('+ Summary information in excel file: ', name_excel)
writer = pd.ExcelWriter(name_excel,
engine='xlsxwriter') ## open excel handle
## KMA dataframe: print result for sources
results_summary_KMA = dataFrame_kma[[
'Sample', '#Template', 'Query_Coverage', 'Template_Coverage', 'Depth',
'Database'
]]
## Sum plasmid and chromosome statistics ##
## sum coverage
total_coverage = results_summary_KMA.groupby(
'Sample')['Query_Coverage'].sum().reset_index()
## debug message
if (Debug):
print("*** Sum: Query_coverage ***")
print(total_coverage)
## TODO: FIX SUMMARY REPORT
results_summary_KMA = results_summary_KMA.set_index('Sample')
results_summary_KMA = results_summary_KMA.sort_values(
by=['Sample', 'Database', 'Query_Coverage'],
ascending=[True, True, True])
results_summary_KMA.to_excel(writer,
sheet_name='KMA') ## write excel handle
## write MLST
if (MLST_results):
MLST_all.to_excel(writer, sheet_name='MLST')
## write excel and close
writer.save() ## close excel handle
print("\n+ Check summary of results in file generated")
### timestamp
start_time_partial = HCGB_time.timestamp(start_time_partial)
######################################
## update database for later usage
######################################
if not options.fast:
HCGB_aes.boxymcboxface("Update Sample Database")
## update db
print("+ Update database with samples identified")
## debug message
if (Debug):
print(colored("**DEBUG: dataFrame_edirect **", 'yellow'))
pd.set_option('display.max_colwidth', None)
pd.set_option('display.max_columns', None)
print(dataFrame_edirect)
## dataFrame_edirect
file_toprint = final_dir + '/edirect_info2download.csv'
dataFrame_edirect.to_csv(file_toprint)
## update database with samples identified
data2download = dataFrame_edirect.filter(
['genus', 'species', 'strain', 'genome'])
data2download = data2download.rename(columns={
'genome': 'NCBI_assembly_ID',
'strain': 'name'
})
NCBI_folder = os.path.abspath(options.database) + '/NCBI'
database_generator.NCBI_DB(data2download, NCBI_folder, Debug)
else:
print(
"+ No update of the database has been requested using option --fast"
)
print("\n*************** Finish *******************")
start_time_partial = HCGB_time.timestamp(start_time_total)
print("+ Exiting identification module.")
return ()
