def bin_fastqa_file(fq_fa_lst, tax_annot_res_dir, sens, n_thr, min_qual,
min_qlen, min_pident, min_coverage, no_trash):
# Function for parallel binning FASTQ and FASTA files.
# Actually bins multiple files.
#
# :param fq_fa_lst: lsit of paths to FASTQ (of FASTA) file meant to be processed;
# :type fq_fa_lst: list<str>;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
for fq_fa_path in fq_fa_lst:
new_dpath = get_curr_res_dpath(fq_fa_path, tax_annot_res_dir)
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy",
"taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens,
taxonomy_path)
# Configure path to trash file
if is_fastq(fq_fa_path):
seq_records_generator = fastq_records
write_fun = write_fastq_record
else:
seq_records_generator = fasta_records
write_fun = write_fasta_record
# end if
# Make filter for quality and length
QL_filter = get_QL_filter(fq_fa_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
fq_fa_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(fq_fa_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# Create an iterator that will yield records
seq_records_iterator = iter(seq_records_generator(fq_fa_path))
# Dict for storing batches of sequences meant to be written to output files:
to_write = dict()
stop = False # for outer while-loop
while not stop:
# Extract batch of records of 'n_thr' size and find their destination paths:
for _ in range(n_thr):
try:
fastqa_rec = next(seq_records_iterator)
except StopIteration:
stop = True # for outer while-loop
break
# end try
read_name = sys.intern(fmt_read_id(
fastqa_rec["seq_id"])[1:]) # get ID of the sequence
try:
hit_names, *vals_to_filter = resfile_lines[
read_name] # find hit corresponding to this sequence
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(read_name))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error(
"Make sure that this read has been already processed by \
`barapost-prober.py` and `barapost-local.py`.")
platf_depend_exit(1)
# end try
# If read is found in TSV file:
if not QL_filter(vals_to_filter):
# Place this sequence to QL trash file
to_write[read_name] = (fastqa_rec, QL_trash_fpath)
QL_seqs_fail += 1
elif not align_filter(vals_to_filter):
# Place this sequence to QL trash file
to_write[read_name] = (fastqa_rec, align_trash_fpath)
align_seqs_fail += 1
else:
for hit_name in hit_names.split("&&"):
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path, "{}.fast{}".format(
hit_name,
'q' if is_fastq(fq_fa_path) else 'a'))
to_write[read_name] = (fastqa_rec, binned_file_path)
# end for
seqs_pass += 1
# end if
# end for
# Write batch of records to output files:
with write_lock:
for record, fpath in to_write.values():
write_fun(fpath, record)
# end for
# end with
to_write.clear()
# end while
with write_lock:
# Write the rest of 'uneven' data to output files:
if len(to_write) != 0:
for record, fpath in to_write.values():
write_fun(fpath, record)
# end for
# end if
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(fq_fa_path)))
printn(" Working...")
# end with
# end for
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def bin_fastqa_file(fq_fa_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function for single-thread binning FASTQ and FASTA files.
#
# :param fq_fa_path: path to FASTQ (of FASTA) file meant to be processed;
# :type fq_fa_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict(
) # dict containing file objects of existing output files
new_dpath = get_curr_res_dpath(fq_fa_path, tax_annot_res_dir)
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy", "taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens, taxonomy_path)
# Configure generator, write function and path to trash file
if is_fastq(fq_fa_path):
seq_records_generator = fastq_records
write_fun = write_fastq_record
else:
seq_records_generator = fasta_records
write_fun = write_fasta_record
# end if
# Make filter for quality and length
QL_filter = get_QL_filter(fq_fa_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
fq_fa_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(fq_fa_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
for fastq_rec in seq_records_generator(fq_fa_path):
read_name = sys.intern(fmt_read_id(
fastq_rec["seq_id"])[1:]) # get ID of the sequence
try:
hit_names, *vals_to_filter = resfile_lines[
read_name] # find hit corresponding to this sequence
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(read_name))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error("Make sure that this read has been already \
processed by `barapost-prober.py` and `barapost-local.py`.")
platf_depend_exit(1)
# end try
# Apply filters
if not QL_filter(vals_to_filter):
QL_seqs_fail += 1
# Place this sequence to QL trash file
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, QL_trash_fpath)
# end if
write_fun(srt_file_dict[QL_trash_fpath],
fastq_rec) # write current read to binned file
elif not align_filter(vals_to_filter):
align_seqs_fail += 1
# Place this sequence to align_trash file
if align_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
align_trash_fpath)
# end if
write_fun(srt_file_dict[align_trash_fpath],
fastq_rec) # write current read to binned file
else:
for hit_name in hit_names.split(
"&&"
): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path,
"{}.fast{}".format(hit_name,
'q' if is_fastq(fq_fa_path) else 'a'))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
binned_file_path)
# end if
write_fun(srt_file_dict[binned_file_path],
fastq_rec) # write current read to binned file
# end for
seqs_pass += 1
# end if
# end for
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(fq_fa_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def bin_fast5_file(f5_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function bins FAST5 file with untwisting.
#
# :param f5_path: path to FAST5 file meant to be processed;
# :type f5_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict()
index_dirpath = os.path.join(
tax_annot_res_dir,
index_name) # name of directory that will contain indicies
# Make filter for quality and length
QL_filter = get_QL_filter(f5_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
f5_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(f5_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# File validation:
# RuntimeError will be raised if FAST5 file is broken.
try:
# File existance checking is performed while parsing CL arguments.
# Therefore, this if-statement will trigger only if f5_path's file is not a valid HDF5 file.
if not h5py.is_hdf5(f5_path):
raise RuntimeError("file is not of HDF5 (i.e. not FAST5) format")
# end if
from_f5 = h5py.File(f5_path, 'r')
for _ in from_f5:
break
# end for
except RuntimeError as runterr:
printlog_error_time("FAST5 file is broken")
printlog_error("Reading the file `{}` crashed.".format(
os.path.basename(f5_path)))
printlog_error("Reason: {}".format(str(runterr)))
printlog_error("Omitting this file...")
print()
# Return zeroes -- inc_val won't be incremented and this file will be omitted
return (0, 0, 0)
# end try
# singleFAST5 and multiFAST5 files should be processed in different ways
# "Raw" group always in singleFAST5 root and never in multiFAST5 root
if "Raw" in from_f5.keys():
f5_cpy_func = copy_single_f5
else:
f5_cpy_func = copy_read_f5_2_f5
# end if
readids_to_seek = list(from_f5.keys()) # list of not-binned-yet read IDs
# Fill the list 'readids_to_seek'
for read_name in fast5_readids(from_f5):
# Get rid of "read_"
readids_to_seek.append(sys.intern(read_name))
# end for
# Walk through the index
index_f5_2_tsv = open_shelve(os.path.join(index_dirpath, index_name), 'r')
if not f5_path in index_f5_2_tsv.keys():
printlog_error_time(
"Source FAST5 file `{}` not found in index".format(f5_path))
printlog_error("Try to rebuild index")
platf_depend_exit(1)
# end if
for tsv_path in index_f5_2_tsv[f5_path].keys():
read_names = index_f5_2_tsv[f5_path][tsv_path]
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy",
"taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_path, sens, taxonomy_path)
for read_name in read_names:
try:
hit_names, *vals_to_filter = resfile_lines[sys.intern(
fmt_read_id(read_name)[1:])]
except KeyError:
printlog_error_time("Error: missing taxonomic annotation info for read `{}`"\
.format(fmt_read_id(read_name)[1:]))
printlog_error(
"It is stored in `{}` FAST5 file".format(f5_path))
printlog_error(
"Try to make new index file (press ENTER on corresponding prompt)."
)
printlog_error(
"Or, if does not work for you, make sure that taxonomic annotation info \
for this read is present in one of TSV files generated by `barapost-prober.py` and `barapost-local.py`."
)
index_f5_2_tsv.close()
platf_depend_exit(1)
# end try
if not QL_filter(vals_to_filter):
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
QL_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[QL_trash_fpath])
QL_seqs_fail += 1
elif not align_filter(vals_to_filter):
# Get name of result FASTQ file to write this read in
if align_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
align_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name,
srt_file_dict[align_trash_fpath])
align_seqs_fail += 1
else:
for hit_name in hit_names.split(
"&&"
): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path, "{}.fast5".format(hit_name))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(
srt_file_dict, binned_file_path)
# end if
f5_cpy_func(from_f5, read_name,
srt_file_dict[binned_file_path])
# end for
seqs_pass += 1
# end if
# end for
from_f5.close()
index_f5_2_tsv.close()
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(f5_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def bin_fast5_file(f5_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function bins FAST5 file without untwisting.
#
# :param f5_path: path to FAST5 file meant to be processed;
# :type f5_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict()
new_dpath = glob("{}{}*{}*".format(tax_annot_res_dir, os.sep, get_checkstr(f5_path)))[0]
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy", "taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens, taxonomy_path)
# Make filter for quality and length
QL_filter = get_QL_filter(f5_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(f5_path, outdir_path, min_qual, min_qlen,)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(f5_path, outdir_path, min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# File validation:
# RuntimeError will be raised if FAST5 file is broken.
try:
# File existance checking is performed while parsing CL arguments.
# Therefore, this if-statement will trigger only if f5_path's file is not a valid HDF5 file.
if not h5py.is_hdf5(f5_path):
raise RuntimeError("file is not of HDF5 (i.e. not FAST5) format")
# end if
from_f5 = h5py.File(f5_path, 'r')
for _ in from_f5:
break
# end for
except RuntimeError as runterr:
printlog_error_time("FAST5 file is broken")
printlog_error("Reading the file `{}` crashed.".format(os.path.basename(f5_path)))
printlog_error("Reason: {}".format( str(runterr) ))
printlog_error("Omitting this file...")
print()
# Return zeroes -- inc_val won't be incremented and this file will be omitted
return (0, 0, 0)
# end try
# singleFAST5 and multiFAST5 files should be processed in different ways
# "Raw" group always in singleFAST5 root and never in multiFAST5 root
if "Raw" in from_f5.keys():
f5_cpy_func = copy_single_f5
else:
f5_cpy_func = copy_read_f5_2_f5
# end if
for _, read_name in enumerate(fast5_readids(from_f5)):
try:
hit_names, *vals_to_filter = resfile_lines[sys.intern(fmt_read_id(read_name))[1:]] # omit 'read_' in the beginning of FAST5 group's name
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(fmt_read_id(read_name)))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error("Try running barapost-binning with `-u` (`--untwist-fast5`) flag.\n")
platf_depend_exit(1)
# end try
# If read is found in TSV file:
if not QL_filter(vals_to_filter):
QL_seqs_fail += 1
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, QL_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[QL_trash_fpath])
elif not align_filter(vals_to_filter):
align_seqs_fail += 1
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, align_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[align_trash_fpath])
else:
for hit_name in hit_names.split("&&"): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(outdir_path, "{}.fast5".format(hit_name))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, binned_file_path)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[binned_file_path])
# end for
seqs_pass += 1
# end if
# end for
from_f5.close()
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(os.path.basename(f5_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)