def pass_processed_seqs(fasta_file, num_done_seqs, fmt_func):
# Function passes sequences that have been already processed.
# :param fasta_file: FASTA file instalce;
# :type fasta_file: str;
# :param num_done_seqs: amount of sequences that have been already processed;
# :type num_done_seqs: int;
# :param fmt_func: function from 'FORMATTING_FUNCS' tuple;
if num_done_seqs == 0:
return None # no sequences have been processed
else:
i = 0
while i <= num_done_seqs:
line = fmt_func(fasta_file.readline())
if line == "":
return ""
# end if
if line.startswith('>'):
line = fmt_read_id(line)
next_id_line = line
i += 1
# end if
# end while
# return ID of seqeunce, which will be included in the next packet first
return next_id_line
def fast5_readids(fast5_file):
# Generator yields IDs of all reads in a FAST5 file.
# :param fast5_file: object of HDF5 FAST5 file;
# :type fast5_file: h5py.File;
if "Raw" in fast5_file.keys(): # single-FAST5 file
yield "read_" + fmt_read_id(fast5_file.filename) # name of read is in filename
else: # multi-FAST5 file
for readid in fast5_file:
if readid.startswith("read_"):
yield readid
def form_packet_totalbp(fastq_file, packet_size, fmt_func, max_seq_len):
# Function reads lines from 'fastq_file' and composes a packet of 'packet_size' base pairs.
# :param fastq_file: file instance from which to read;
# :type fastq_file: _io.TextIOWrapper or gzip.File;
# :param packet_size: number of base pairs to retrive from file;
# :type packet_size: int;
# :param fmt_func: formating functio nfrom FORMMATING_FUNCS tuple;
# :param max_seq_len: maximum length of a sequence proessed;
# :type max_seq_len: int (None if pruning is disabled);
packet = ""
qual_dict = dict() # {<seq_id>: <read_quality>}
eof = False
totalbp = 0
while totalbp < packet_size:
try:
read_id = fmt_func(fastq_file.readline())
except UnicodeDecodeError as err:
print()
printlog_warning("Warning: current file is broken: {}."\
.format(str(err)))
printlog_warning("File: `{}`".format(os.path.abspath(fastq_file.name)))
printlog_warning("Ceasing reading sequences from this file.")
eof = True
break
# end try
if read_id == "": # if eof is reached, leave now
eof = True
break
# end if
read_id = fmt_read_id(read_id)
seq = fmt_func(fastq_file.readline())
fastq_file.readline() # pass comment
avg_qual = get_read_avg_qual( fmt_func(fastq_file.readline()) )
packet += read_id + '\n' + seq + '\n'
qual_dict[read_id[1:]] = avg_qual
totalbp += min(len(seq), max_seq_len)
# end while
if max_seq_len < float("inf"): # prune sequences
packet = prune_seqs(packet, max_seq_len)
# end if
return {"fasta": packet, "qual": qual_dict}, eof
def bin_fastqa_file(fq_fa_lst, tax_annot_res_dir, sens, n_thr, min_qual,
min_qlen, min_pident, min_coverage, no_trash):
# Function for parallel binning FASTQ and FASTA files.
# Actually bins multiple files.
#
# :param fq_fa_lst: lsit of paths to FASTQ (of FASTA) file meant to be processed;
# :type fq_fa_lst: list<str>;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
for fq_fa_path in fq_fa_lst:
new_dpath = get_curr_res_dpath(fq_fa_path, tax_annot_res_dir)
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy",
"taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens,
taxonomy_path)
# Configure path to trash file
if is_fastq(fq_fa_path):
seq_records_generator = fastq_records
write_fun = write_fastq_record
else:
seq_records_generator = fasta_records
write_fun = write_fasta_record
# end if
# Make filter for quality and length
QL_filter = get_QL_filter(fq_fa_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
fq_fa_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(fq_fa_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# Create an iterator that will yield records
seq_records_iterator = iter(seq_records_generator(fq_fa_path))
# Dict for storing batches of sequences meant to be written to output files:
to_write = dict()
stop = False # for outer while-loop
while not stop:
# Extract batch of records of 'n_thr' size and find their destination paths:
for _ in range(n_thr):
try:
fastqa_rec = next(seq_records_iterator)
except StopIteration:
stop = True # for outer while-loop
break
# end try
read_name = sys.intern(fmt_read_id(
fastqa_rec["seq_id"])[1:]) # get ID of the sequence
try:
hit_names, *vals_to_filter = resfile_lines[
read_name] # find hit corresponding to this sequence
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(read_name))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error(
"Make sure that this read has been already processed by \
`barapost-prober.py` and `barapost-local.py`.")
platf_depend_exit(1)
# end try
# If read is found in TSV file:
if not QL_filter(vals_to_filter):
# Place this sequence to QL trash file
to_write[read_name] = (fastqa_rec, QL_trash_fpath)
QL_seqs_fail += 1
elif not align_filter(vals_to_filter):
# Place this sequence to QL trash file
to_write[read_name] = (fastqa_rec, align_trash_fpath)
align_seqs_fail += 1
else:
for hit_name in hit_names.split("&&"):
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path, "{}.fast{}".format(
hit_name,
'q' if is_fastq(fq_fa_path) else 'a'))
to_write[read_name] = (fastqa_rec, binned_file_path)
# end for
seqs_pass += 1
# end if
# end for
# Write batch of records to output files:
with write_lock:
for record, fpath in to_write.values():
write_fun(fpath, record)
# end for
# end with
to_write.clear()
# end while
with write_lock:
# Write the rest of 'uneven' data to output files:
if len(to_write) != 0:
for record, fpath in to_write.values():
write_fun(fpath, record)
# end for
# end if
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(fq_fa_path)))
printn(" Working...")
# end with
# end for
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def fasta_packets(fasta,
packet_size,
num_done_seqs,
packet_mode=0,
saved_packet_size=None,
saved_packet_mode=None,
max_seq_len=float("inf"),
probing_batch_size=float("inf")):
# Generator yields fasta-formattedpackets of records from fasta file.
# This function passes 'num_done_seqs' sequences (i.e. they will not be processed)
# to 'pass_processed_files'.
#
# :param fasta: path to fasta file;
# :type fasta: str;
# :param packet_size: number of sequences to align in one request ('blastn' launching);
# :type packet_size: int;
# :param num_done_seqs: number of sequnces in current file that have been already processed;
# :type num_done_seqs: int;
# :param packet_mode: packet mode (see -c option);
# :type packet_mode: int;
# :param saved_packet_size: size of last sent packet from tmp file. Necessary for resumption.
# It will be None, if no tmp file was in classification directory;
# :type saved_packet_size: int;
# :param saved_packet_mode: mode used whilst formig the last sent packet from tmp file.
# Necessary for resumption. It will be None, if no tmp file was in classification directory;
# :type saved_packet_mode: int;
# :param max_seq_len: maximum length of a sequence proessed;
# :type max_seq_len: int (float("inf") if pruning is disabled);
how_to_open = OPEN_FUNCS[is_gzipped(fasta)]
fmt_func = FORMATTING_FUNCS[is_gzipped(fasta)]
with how_to_open(fasta) as fasta_file:
# Next line retrieving is implemented as simple line-from-file reading.
get_next_line = lambda: fmt_func(fasta_file.readline())
# Variable that contains ID of next sequence in current FASTA file.
# If no or all sequences in current FASTA file have been already processed, this variable is None.
# There is no way to count sequences in multi-FASTA file, accept of counting sequence IDs.
# Therefore 'next_id_line' should be saved in memory just after moment when packet is formed.
next_id_line = pass_processed_seqs(fasta_file, num_done_seqs, fmt_func)
if next_id_line == "":
yield {"fasta": "", "qual": dict()}
# end if
packet = ""
# We are resuming, nucleotide sequence will be saved in 'line' variable here:
try:
line = get_next_line()
except UnicodeDecodeError as err:
print()
printlog_warning("Warning: current file is broken: {}."\
.format(str(err)))
printlog_warning("File: `{}`".format(fasta))
printlog_warning("Ceasing reading sequences from this file.")
return
# end try
if line.startswith('>'):
line = fmt_read_id(line) # format sequence ID
# end if
# If some sequences have been passed, this if-statement will be executed.
# New packet should start with sequence ID line.
if not next_id_line is None:
packet += next_id_line + '\n'
# end if
packet += line + '\n' # add recently read line
# Here goes check for saved packet size and mode:
if not saved_packet_size is None:
wrk_pack_size = saved_packet_size
else:
wrk_pack_size = packet_size
# end if
if not saved_packet_mode is None:
wrk_pack_mode = saved_packet_mode
else:
wrk_pack_mode = packet_mode
# end if
eof = False
while not eof: # till the end of file
counter = 0 # variable for counting sequences within packet
seqlen = 0
while counter < wrk_pack_size:
try:
line = get_next_line()
except UnicodeDecodeError as err:
print()
printlog_warning("Warning: current file is broken: {}."\
.format(str(err)))
printlog_warning("File: `{}`".format(fasta))
printlog_warning(
"Ceasing reading sequences from this file.")
line = ""
break
# end try
if line.startswith('>'):
line = fmt_read_id(line)
if packet_mode == 0:
counter += 1
else:
counter += min(seqlen, max_seq_len)
seqlen = 0
# end if
# end if
if line == "": # if end of file (data) is reached
break
# end if
if not line.startswith('>'):
seqlen += len(line.strip())
# end if
packet += line + '\n' # add line to packet
# end while
if line != "":
next_id_line = packet.splitlines()[
-1] # save sequence ID next packet will start with
packet = '\n'.join(packet.splitlines()
[:-1]) # exclude 'next_id_line' from packet
else:
eof = True
next_id_line = None
# end if
# Get list of sequence IDs:
names = filter(lambda l: l.startswith('>'), packet.splitlines())
names = map(lambda l: l.replace('>', ''), names)
# {<seq_id>: '-'}, as soon as it is a fasta file
qual_dict = {name: '-' for name in names}
if max_seq_len < float("inf"): # prune sequences
packet = prune_seqs(packet, max_seq_len)
# end if
if packet != "":
yield {"fasta": packet, "qual": qual_dict}
if packet_mode == 0:
probing_batch_size -= wrk_pack_size
wrk_pack_size = min(packet_size, probing_batch_size)
else:
probing_batch_size -= len(qual_dict)
# end if
# Switch back to standart packet size
# As Vorotos said, repeated assignment is the best check:
if wrk_pack_mode != packet_mode:
wrk_pack_mode = packet_mode
# end if
if not next_id_line is None:
packet = next_id_line + '\n'
# end if
else:
return
def fasta_packets_from_str(data, packet_size):
# Generator retrieves 'packet_size' records from 'fasta'
# no matter whether is it path to FASTA file of actual FASTA data of 'str' type.
# :param data: FASTA-formatted string;
# :type data: str;
# :param packet_size: number of sequences to align in one 'blastn' launching;
# :type packet_size: int;
fasta_lines = data.splitlines()
fasta_lines.append("") # append fictive value imitating end of file
del data # let interpreter get rid of this large string -- we do not need it any more
qual_dict = dict(
) # {<seq_id>: '-'}, as soon as fasta file is being processed
# Variable for counting lines (it is list in order to circumvent immutability of int type)
line_i = 0
# Variable that contains id of next sequence in current FASTA file.
# If no or all sequences in current FASTA file have been already processed, this variable is None.
# There is no way to count sequences in multi-FASTA file, accept of counting sequence IDs.
# Therefore 'next_id_line' should be saved in memory after moment when packet is formed.
next_id_line = None
# The first line is always a sequence ID if data is not a file, but a fasta-formatted string.
# Because in this case all "done" sequences are already passed by function 'fasta_packets'
line = fmt_read_id(fasta_lines[line_i])
line_i += 1
packet = line + '\n' # add recently read line
eof = False
# Iterate over packets left to process
while not eof:
i = 0 # variable for counting sequenes within packet
while i < packet_size:
line = fasta_lines[line_i]
if line == "": # if end of data is reached
eof = True
break
# end if
line_i += 1
if line.startswith('>'):
line = fmt_read_id(line)
i += 1
# end if
packet += line + '\n' # add line to packet
# end while
if line != "":
next_id_line = packet.splitlines()[
-1] # save sequence ID next packet will start with
packet = '\n'.join(
packet.splitlines()[:-1]) # exclude 'next_id_line' from packet
else:
next_id_line = None
# end if
# Get list of sequence IDs:
names = filter(lambda l: l.startswith('>'), packet.splitlines())
names = map(lambda l: l.replace('>', ''), names)
for name in names:
qual_dict[name] = '-' # there is no quality info in fasta files
# end for
# No way to get quality from fasta-formatted string.
# However, we will have it from 'packet_generator()' launching
# (see function 'process' in src/barapost_local_modules/parallel_single_file.py).
if packet != "":
yield {"fasta": packet, "qual": qual_dict}
# Reset packet
if not next_id_line is None:
packet = next_id_line + '\n'
qual_dict = dict()
# end if
else:
return
def map_f5reads_2_taxann(f5_path, tsv_taxann_lst, tax_annot_res_dir):
# Function perform mapping of all reads stored in input FAST5 files
# to existing TSV files containing taxonomic annotation info.
#
# It creates an DBM index file.
#
# :param f5_path: path to current FAST5 file;
# :type f5_path: str;
# :param tsv_taxann_lst: list of path to TSV files that contain taxonomic annotation;
# :type tsv_taxann_lst: list<str>;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
index_dirpath = os.path.join(
tax_annot_res_dir,
index_name) # name of directory that will contain indicies
# File validation:
# RuntimeError will be raised if FAST5 file is broken.
try:
# File existance checking is performed while parsing CL arguments.
# Therefore, this if-statement will trigger only if f5_path's file is not a valid HDF5 file.
if not h5py.is_hdf5(f5_path):
raise RuntimeError("file is not of HDF5 (i.e. not FAST5) format")
# end if
f5_file = h5py.File(f5_path, 'r')
for _ in f5_file:
break
# end for
except RuntimeError as runterr:
printlog_error_time("FAST5 file is broken")
printlog_error("Reading the file `{}` crashed.".format(
os.path.basename(f5_path)))
printlog_error("Reason: {}".format(str(runterr)))
printlog_error("Omitting this file...")
print()
return
# end try
readids_to_seek = list(fast5_readids(f5_file))
idx_dict = dict() # dictionary for index
# This saving is needed to compare with 'len(readids_to_seek)'
# after all TSV will be looked through in order to
# determine if some reads miss taxonomic annotation.
len_before = len(readids_to_seek)
# Iterate over TSV-taaxnn file
for tsv_taxann_fpath in tsv_taxann_lst:
with open(tsv_taxann_fpath, 'r') as taxann_file:
# Get all read IDs in current TSV
readids_in_tsv = list(
map(lambda l: l.split('\t')[0], taxann_file.readlines()))
# Iterate over all other reads in current FAST5
# ('reversed' is necessary because we remove items from list in this loop)
for readid in reversed(readids_to_seek):
fmt_id = fmt_read_id(readid)[1:]
if fmt_id in readids_in_tsv:
# If not first -- write data to dict (and to index later)
try:
idx_dict[tsv_taxann_fpath].append(
"read_" + fmt_id) # append to existing list
except KeyError:
idx_dict[tsv_taxann_fpath] = ["read_" + fmt_id
] # create a new list
finally:
readids_to_seek.remove(readid)
# end try
# end if
# end for
# end with
if len(readids_to_seek) == 0:
break
# end if
# end for
# If after all TSV is checked but nothing have changed -- we miss taxonomic annotation
# for some reads! And we will write their IDs to 'missing_reads_lst.txt' file.
if len(readids_to_seek) == len_before:
printlog_error_time("reads from FAST5 file not found")
printlog_error("FAST5 file: `{}`".format(f5_path))
printlog_error("Some reads have not undergone taxonomic annotation.")
missing_log = "missing_reads_lst.txt"
printlog_error("List of missing reads are in following file:")
printlog_error("{}".format(missing_log))
with open(missing_log, 'w') as missing_logfile:
missing_logfile.write(
"Missing reads from file '{}':\n\n".format(f5_path))
for readid in readids_to_seek:
missing_logfile.write(fmt_read_id(readid) + '\n')
# end for
try:
for path in glob(os.path.join(index_dirpath, '*')):
os.unlink(path)
# end for
os.rmdir(index_dirpath)
except OSError as oserr:
printlog_error(
"Error occured while removing index directory: {}".format(
oserr))
finally:
platf_depend_exit(3)
# end try
# end if
try:
# Open index files appending to existing data ('c' parameter)
with open_shelve(os.path.join(index_dirpath, index_name),
'c') as index_f5_2_tsv:
# Update index
index_f5_2_tsv[f5_path] = idx_dict
# end with
except OSError as oserr:
printlog_error_time("Error: cannot create index file `{}`"\
.format(os.path.join(index_dirpath, index_name)))
printlog_error(str(oserr))
platf_depend_exit(1)
def bin_fast5_file(f5_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function bins FAST5 file with untwisting.
#
# :param f5_path: path to FAST5 file meant to be processed;
# :type f5_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict()
index_dirpath = os.path.join(
tax_annot_res_dir,
index_name) # name of directory that will contain indicies
# Make filter for quality and length
QL_filter = get_QL_filter(f5_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
f5_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(f5_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# File validation:
# RuntimeError will be raised if FAST5 file is broken.
try:
# File existance checking is performed while parsing CL arguments.
# Therefore, this if-statement will trigger only if f5_path's file is not a valid HDF5 file.
if not h5py.is_hdf5(f5_path):
raise RuntimeError("file is not of HDF5 (i.e. not FAST5) format")
# end if
from_f5 = h5py.File(f5_path, 'r')
for _ in from_f5:
break
# end for
except RuntimeError as runterr:
printlog_error_time("FAST5 file is broken")
printlog_error("Reading the file `{}` crashed.".format(
os.path.basename(f5_path)))
printlog_error("Reason: {}".format(str(runterr)))
printlog_error("Omitting this file...")
print()
# Return zeroes -- inc_val won't be incremented and this file will be omitted
return (0, 0, 0)
# end try
# singleFAST5 and multiFAST5 files should be processed in different ways
# "Raw" group always in singleFAST5 root and never in multiFAST5 root
if "Raw" in from_f5.keys():
f5_cpy_func = copy_single_f5
else:
f5_cpy_func = copy_read_f5_2_f5
# end if
readids_to_seek = list(from_f5.keys()) # list of not-binned-yet read IDs
# Fill the list 'readids_to_seek'
for read_name in fast5_readids(from_f5):
# Get rid of "read_"
readids_to_seek.append(sys.intern(read_name))
# end for
# Walk through the index
index_f5_2_tsv = open_shelve(os.path.join(index_dirpath, index_name), 'r')
if not f5_path in index_f5_2_tsv.keys():
printlog_error_time(
"Source FAST5 file `{}` not found in index".format(f5_path))
printlog_error("Try to rebuild index")
platf_depend_exit(1)
# end if
for tsv_path in index_f5_2_tsv[f5_path].keys():
read_names = index_f5_2_tsv[f5_path][tsv_path]
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy",
"taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_path, sens, taxonomy_path)
for read_name in read_names:
try:
hit_names, *vals_to_filter = resfile_lines[sys.intern(
fmt_read_id(read_name)[1:])]
except KeyError:
printlog_error_time("Error: missing taxonomic annotation info for read `{}`"\
.format(fmt_read_id(read_name)[1:]))
printlog_error(
"It is stored in `{}` FAST5 file".format(f5_path))
printlog_error(
"Try to make new index file (press ENTER on corresponding prompt)."
)
printlog_error(
"Or, if does not work for you, make sure that taxonomic annotation info \
for this read is present in one of TSV files generated by `barapost-prober.py` and `barapost-local.py`."
)
index_f5_2_tsv.close()
platf_depend_exit(1)
# end try
if not QL_filter(vals_to_filter):
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
QL_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[QL_trash_fpath])
QL_seqs_fail += 1
elif not align_filter(vals_to_filter):
# Get name of result FASTQ file to write this read in
if align_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
align_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name,
srt_file_dict[align_trash_fpath])
align_seqs_fail += 1
else:
for hit_name in hit_names.split(
"&&"
): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path, "{}.fast5".format(hit_name))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(
srt_file_dict, binned_file_path)
# end if
f5_cpy_func(from_f5, read_name,
srt_file_dict[binned_file_path])
# end for
seqs_pass += 1
# end if
# end for
from_f5.close()
index_f5_2_tsv.close()
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(f5_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def bin_fastqa_file(fq_fa_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function for single-thread binning FASTQ and FASTA files.
#
# :param fq_fa_path: path to FASTQ (of FASTA) file meant to be processed;
# :type fq_fa_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(
logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict(
) # dict containing file objects of existing output files
new_dpath = get_curr_res_dpath(fq_fa_path, tax_annot_res_dir)
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy", "taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens, taxonomy_path)
# Configure generator, write function and path to trash file
if is_fastq(fq_fa_path):
seq_records_generator = fastq_records
write_fun = write_fastq_record
else:
seq_records_generator = fasta_records
write_fun = write_fasta_record
# end if
# Make filter for quality and length
QL_filter = get_QL_filter(fq_fa_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(
fq_fa_path,
outdir_path,
min_qual,
min_qlen,
)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(fq_fa_path, outdir_path,
min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
for fastq_rec in seq_records_generator(fq_fa_path):
read_name = sys.intern(fmt_read_id(
fastq_rec["seq_id"])[1:]) # get ID of the sequence
try:
hit_names, *vals_to_filter = resfile_lines[
read_name] # find hit corresponding to this sequence
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(read_name))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error("Make sure that this read has been already \
processed by `barapost-prober.py` and `barapost-local.py`.")
platf_depend_exit(1)
# end try
# Apply filters
if not QL_filter(vals_to_filter):
QL_seqs_fail += 1
# Place this sequence to QL trash file
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, QL_trash_fpath)
# end if
write_fun(srt_file_dict[QL_trash_fpath],
fastq_rec) # write current read to binned file
elif not align_filter(vals_to_filter):
align_seqs_fail += 1
# Place this sequence to align_trash file
if align_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
align_trash_fpath)
# end if
write_fun(srt_file_dict[align_trash_fpath],
fastq_rec) # write current read to binned file
else:
for hit_name in hit_names.split(
"&&"
): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(
outdir_path,
"{}.fast{}".format(hit_name,
'q' if is_fastq(fq_fa_path) else 'a'))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict,
binned_file_path)
# end if
write_fun(srt_file_dict[binned_file_path],
fastq_rec) # write current read to binned file
# end for
seqs_pass += 1
# end if
# end for
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(
os.path.basename(fq_fa_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
def bin_fast5_file(f5_path, tax_annot_res_dir, sens, min_qual, min_qlen,
min_pident, min_coverage, no_trash):
# Function bins FAST5 file without untwisting.
#
# :param f5_path: path to FAST5 file meant to be processed;
# :type f5_path: str;
# :param tax_annot_res_dir: path to directory containing taxonomic annotation;
# :type tax_annot_res_dir: str;
# :param sens: binning sensitivity;
# :type sens: str;
# :param min_qual: threshold for quality filter;
# :type min_qual: float;
# :param min_qlen: threshold for length filter;
# :type min_qlen: int (or None, if this filter is disabled);
# :param min_pident: threshold for alignment identity filter;
# :type min_pident: float (or None, if this filter is disabled);
# :param min_coverage: threshold for alignment coverage filter;
# :type min_coverage: float (or None, if this filter is disabled);
# :param no_trash: loical value. True if user does NOT want to output trash files;
# :type no_trash: bool;
outdir_path = os.path.dirname(logging.getLoggerClass().root.handlers[0].baseFilename)
seqs_pass = 0 # counter for sequences, which pass filters
QL_seqs_fail = 0 # counter for too short or too low-quality sequences
align_seqs_fail = 0 # counter for sequences, which align to their best hit with too low identity or coverage
srt_file_dict = dict()
new_dpath = glob("{}{}*{}*".format(tax_annot_res_dir, os.sep, get_checkstr(f5_path)))[0]
tsv_res_fpath = get_res_tsv_fpath(new_dpath)
taxonomy_path = os.path.join(tax_annot_res_dir, "taxonomy", "taxonomy.tsv")
resfile_lines = configure_resfile_lines(tsv_res_fpath, sens, taxonomy_path)
# Make filter for quality and length
QL_filter = get_QL_filter(f5_path, min_qual, min_qlen)
# Configure path to trash file
if not no_trash:
QL_trash_fpath = get_QL_trash_fpath(f5_path, outdir_path, min_qual, min_qlen,)
else:
QL_trash_fpath = None
# end if
# Make filter for identity and coverage
align_filter = get_align_filter(min_pident, min_coverage)
# Configure path to this trash file
if not no_trash:
align_trash_fpath = get_align_trash_fpath(f5_path, outdir_path, min_pident, min_coverage)
else:
align_trash_fpath = None
# end if
# File validation:
# RuntimeError will be raised if FAST5 file is broken.
try:
# File existance checking is performed while parsing CL arguments.
# Therefore, this if-statement will trigger only if f5_path's file is not a valid HDF5 file.
if not h5py.is_hdf5(f5_path):
raise RuntimeError("file is not of HDF5 (i.e. not FAST5) format")
# end if
from_f5 = h5py.File(f5_path, 'r')
for _ in from_f5:
break
# end for
except RuntimeError as runterr:
printlog_error_time("FAST5 file is broken")
printlog_error("Reading the file `{}` crashed.".format(os.path.basename(f5_path)))
printlog_error("Reason: {}".format( str(runterr) ))
printlog_error("Omitting this file...")
print()
# Return zeroes -- inc_val won't be incremented and this file will be omitted
return (0, 0, 0)
# end try
# singleFAST5 and multiFAST5 files should be processed in different ways
# "Raw" group always in singleFAST5 root and never in multiFAST5 root
if "Raw" in from_f5.keys():
f5_cpy_func = copy_single_f5
else:
f5_cpy_func = copy_read_f5_2_f5
# end if
for _, read_name in enumerate(fast5_readids(from_f5)):
try:
hit_names, *vals_to_filter = resfile_lines[sys.intern(fmt_read_id(read_name))[1:]] # omit 'read_' in the beginning of FAST5 group's name
except KeyError:
printlog_error_time("Error: read `{}` not found in TSV file containing taxonomic annotation."\
.format(fmt_read_id(read_name)))
printlog_error("This TSV file: `{}`".format(tsv_res_fpath))
printlog_error("Try running barapost-binning with `-u` (`--untwist-fast5`) flag.\n")
platf_depend_exit(1)
# end try
# If read is found in TSV file:
if not QL_filter(vals_to_filter):
QL_seqs_fail += 1
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, QL_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[QL_trash_fpath])
elif not align_filter(vals_to_filter):
align_seqs_fail += 1
# Get name of result FASTQ file to write this read in
if QL_trash_fpath not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, align_trash_fpath)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[align_trash_fpath])
else:
for hit_name in hit_names.split("&&"): # there can be multiple hits for single query sequence
# Get name of result FASTQ file to write this read in
binned_file_path = os.path.join(outdir_path, "{}.fast5".format(hit_name))
if binned_file_path not in srt_file_dict.keys():
srt_file_dict = update_file_dict(srt_file_dict, binned_file_path)
# end if
f5_cpy_func(from_f5, read_name, srt_file_dict[binned_file_path])
# end for
seqs_pass += 1
# end if
# end for
from_f5.close()
# Close all binned files
for file_obj in filter(lambda x: not x is None, srt_file_dict.values()):
file_obj.close()
# end for
sys.stdout.write('\r')
printlog_info_time("File `{}` is binned.".format(os.path.basename(f5_path)))
printn(" Working...")
return (seqs_pass, QL_seqs_fail, align_seqs_fail)
