def parallel_blast(self,
blast_command,
seqfile,
database,
outfile=None,
blast_options=None,
split_dir="splited_fasta",
splited_output_dir="splited_output_dir",
evalue=None,
output_format=None,
threads=None,
num_of_seqs_per_scan=None,
combine_output_to_single_file=True,
async_run=False,
external_process_pool=None):
splited_dir = check_path(split_dir)
splited_out_dir = check_path(splited_output_dir)
save_mkdir(splited_dir)
save_mkdir(splited_out_dir)
number_of_files = num_of_seqs_per_scan if num_of_seqs_per_scan else 5 * threads if threads else 5 * self.threads
self.split_fasta(seqfile, splited_dir, num_of_files=number_of_files)
input_list_of_files = sorted(os.listdir(splited_dir))
list_of_files = []
for filename in input_list_of_files:
filename_prefix = split_filename(filename)[1]
input_file = "%s%s" % (splited_dir, filename)
output_file = "%s%s.hits" % (splited_out_dir, filename_prefix)
list_of_files.append((input_file, output_file))
options_list = []
out_files = []
for in_file, out_filename in list_of_files:
options = " -out %s" % out_filename
options += " -db %s" % database
options += " -query %s" % in_file
options += " %s" % blast_options if blast_options else ""
options += " -evalue %s" % evalue if evalue else ""
options += " -outfmt %i" % output_format if output_format else ""
options_list.append(options)
out_files.append(out_filename)
self.parallel_execute(options_list,
cmd=blast_command,
threads=threads,
async_run=async_run,
external_process_pool=external_process_pool)
if combine_output_to_single_file:
CGAS.cat(out_files, output=outfile)
def parallel_convert(self,
list_of_files,
output_directory,
output_format=".tiff"):
save_mkdir(output_directory)
options_list = []
for filename in list_of_files:
option = " %s" % filename
option += " %s%s%s" % (check_path(output_directory),
split_filename(filename)[1], output_format)
options_list.append(option)
self.parallel_execute(options_list)
#parser.add_argument("-t", "--threads", action="store", dest="threads", default=1, type=int,
# help="Number of threads to use in Trimmomatic. Default - 1.")
parser.add_argument("-q", "--average_quality_threshold", action="store", dest="average_quality_threshold", default=15,
type=int,
help="Quality threshold for sliding window. Works only if -q/--average_quality_threshold is set"
"Default - 15.")
parser.add_argument("-u", "--score_type", action="store", dest="score_type", default="phred64",
help="Phred quality score type. Allowed: phred33, phred64. Default: phred64")
parser.add_argument("-n", "--name_type", action="store", dest="name_type", default="short",
help="Type of read name. Required to gather per tile filtering statistics. Default: short")
"""
args = parser.parse_args()
samples = args.samples.split(",") if args.samples else sorted(
os.listdir(args.samples_dir))
save_mkdir(args.output_dir)
overall_stat_file = "%s/overall_samples.stat" % args.output_dir
overall_stat_fd = open(overall_stat_file, "w")
overall_stat_fd.write(
"#Sample_id\tTotal_pairs\tRetained_pairs\tRetained_pairs_percent\tMin_pairs_retained_in_tiles\n"
)
for sample in samples:
print("Handling %s" % sample)
sample_dir = "%s%s/" % (args.samples_dir, sample)
sample_out_dir = "%s%s/" % (args.output_dir, sample)
save_mkdir(sample_out_dir)
files_from_sample_dir = sorted(os.listdir(sample_dir))
action="store",
dest="suffix",
default=".gaps_removed",
help="Suffix to use in output files. Default: '.gaps_removed'")
parser.add_argument("-f",
"--format",
action="store",
dest="format",
default="fasta",
help="Format of alignment")
parser.add_argument("-v",
"--verbose",
action="store_true",
dest="verbose",
help="Print not found ids. Default - no")
args = parser.parse_args()
save_mkdir(args.output)
for alignment_file in args.input:
splited_filename = split_filename(alignment_file)
if args.verbose:
print("Handling %s ..." % alignment_file)
output_filename = "%s%s%s%s" % (args.output, splited_filename[1],
args.suffix, splited_filename[2])
alignment = AlignIO.read(alignment_file, args.format)
filtered_alignment = MultipleAlignmentRoutines.remove_columns_with_gaps(
alignment, args.max_gap_number, gap_symbol=args.gap_symbol)
AlignIO.write(filtered_alignment, output_filename, args.format)
args = parser.parse_args()
Trimmomatic.jar_path = args.path_to_trimmomatic_dir
Trimmomatic.threads = args.threads
#print(Trimmomatic.path)
#print(Trimmomatic.jar_path)
samples = args.samples.split(",") if args.samples else os.listdir(
args.samples_dir)
for sample in samples:
print("Handling %s" % sample)
sample_dir = "%s%s/" % (args.samples_dir, sample)
sample_out_dir = "%s%s/" % (args.output_dir, sample)
save_mkdir(sample_out_dir)
trimmomatic_log = "%s/trimmomatic.log" % sample_out_dir
trimmomatic_time_log = "%s/trimmomatic.time.log" % sample_out_dir
output_prefix = "%s%s.TMF" % (sample_out_dir, sample)
files_from_sample_dir = os.listdir(sample_dir)
left_reads_file = None
right_reads_file = None
for filename in files_from_sample_dir:
if ("_1.fq" in filename) or ("_1.fastq" in filename):
left_reads_file = filename
elif ("_2.fq" in filename) or ("_2.fastq" in filename):
right_reads_file = filename
if (left_reads_file is None) and (right_reads_file is None):
"-s",
"--store_logs",
action="store_true",
dest="store_logs",
default=False,
help="Store download logs in directory set by -g/--logs_dir option")
parser.add_argument("-g",
"--logs_dir",
action="store",
dest="logs_dir",
default="logs",
type=check_path,
help="Directory with logs")
args = parser.parse_args()
save_mkdir(args.output_dir)
save_mkdir(args.logs_dir)
if (not args.alignment) and (not args.tree) and (not args.hmm):
args.all = True
in_fd = sys.stdin if args.input == "stdin" else open(args.input, "r")
family_ids = IdList()
family_ids.read(in_fd)
if args.input != "stdin":
in_fd.close()
absent_alignment_list = IdList()
absent_tree_list = IdList()
def parallel_hmmscan(self,
hmmfile,
seqfile,
outfile,
num_of_seqs_per_scan=None,
split_dir="splited_fasta",
splited_output_dir="splited_output_dir",
tblout_outfile=None,
domtblout_outfile=None,
pfamtblout_outfile=None,
splited_tblout_dir=None,
splited_domtblout_dir=None,
splited_pfamtblout_dir=None,
dont_output_alignments=False,
model_evalue_threshold=None,
model_score_threshold=None,
domain_evalue_threshold=None,
domain_score_threshold=None,
model_evalue_significant_threshold=None,
model_score_significant_threshold=None,
domain_evalue_significant_threshold=None,
domain_score_significant_threshold=None,
use_profile_GA_gathering_cutoffs_for_thresholds=False,
use_profile_NC_noise_cutoffs_for_thresholds=False,
use_profile_TC_trusted_cutoffs_for_thresholds=False,
turn_off_all_heruristics=False,
turn_off_bias_filter=False,
MSV_threshold=None,
Vit_threshold=None,
Fwd_threshold=None,
turn_off_biased_composition_score_corrections=None,
input_format=None,
threads=None,
combine_output_to_single_file=True,
biopython_165_compartibility=False,
remove_tmp_dirs=True,
async_run=False,
external_process_pool=None):
splited_dir = check_path(split_dir)
splited_out_dir = check_path(splited_output_dir)
save_mkdir(splited_dir)
save_mkdir(splited_out_dir)
if splited_tblout_dir:
save_mkdir(splited_tblout_dir)
if splited_domtblout_dir:
save_mkdir(splited_domtblout_dir)
if splited_pfamtblout_dir:
save_mkdir(splited_pfamtblout_dir)
number_of_files = num_of_seqs_per_scan if num_of_seqs_per_scan else 5 * threads if threads else 5 * self.threads
self.split_fasta(seqfile, splited_dir, num_of_files=number_of_files)
input_list_of_files = sorted(os.listdir(splited_dir))
list_of_files = []
for filename in input_list_of_files:
filename_prefix = split_filename(filename)[1]
input_file = "%s%s" % (splited_dir, filename)
output_file = "%s%s.hits" % (splited_out_dir, filename_prefix)
tblout_file = "%s%s.hits" % (splited_tblout_dir, filename_prefix
) if splited_tblout_dir else None
domtblout_file = "%s%s.hits" % (
splited_domtblout_dir,
filename_prefix) if splited_domtblout_dir else None
pfamtblout_file = "%s%s.hits" % (
splited_pfamtblout_dir,
filename_prefix) if splited_pfamtblout_dir else None
list_of_files.append((input_file, output_file, tblout_file,
domtblout_file, pfamtblout_file))
common_options = self.__parse_hmmsxxx_common_options(
tblout=None,
domtblout=None,
pfamtblout=None,
dont_output_alignments=dont_output_alignments,
model_evalue_threshold=model_evalue_threshold,
model_score_threshold=model_score_threshold,
domain_evalue_threshold=domain_evalue_threshold,
domain_score_threshold=domain_score_threshold,
model_evalue_significant_threshold=
model_evalue_significant_threshold,
model_score_significant_threshold=model_score_significant_threshold,
domain_evalue_significant_threshold=
domain_evalue_significant_threshold,
domain_score_significant_threshold=
domain_score_significant_threshold,
use_profile_GA_gathering_cutoffs_for_thresholds=
use_profile_GA_gathering_cutoffs_for_thresholds,
use_profile_NC_noise_cutoffs_for_thresholds=
use_profile_NC_noise_cutoffs_for_thresholds,
use_profile_TC_trusted_cutoffs_for_thresholds=
use_profile_TC_trusted_cutoffs_for_thresholds,
turn_off_all_heruristics=turn_off_all_heruristics,
turn_off_bias_filter=turn_off_bias_filter,
MSV_threshold=MSV_threshold,
Vit_threshold=Vit_threshold,
Fwd_threshold=Fwd_threshold,
turn_off_biased_composition_score_corrections=
turn_off_biased_composition_score_corrections)
common_options += " --qformat %s" if input_format else ""
options_list = []
out_files = []
tblout_files = []
domtblout_files = []
pfamtblout_files = []
for in_file, out_filename, tblout_file, domtblout_file, pfamtblout_file in list_of_files:
options = common_options
options += " --tblout %s" % tblout_file if tblout_file else ""
options += " --domtblout %s" % domtblout_file if domtblout_file else ""
options += " --pfamtblout %s" % pfamtblout_file if pfamtblout_file else ""
options += " -o %s" % out_filename
options += " %s" % hmmfile
options += " %s" % in_file
options_list.append(options)
out_files.append(out_filename)
tblout_files.append(tblout_file)
domtblout_files.append(domtblout_file)
pfamtblout_files.append(pfamtblout_file)
self.parallel_execute(options_list,
cmd="hmmscan",
threads=threads,
async_run=async_run,
external_process_pool=external_process_pool)
if combine_output_to_single_file:
if biopython_165_compartibility:
CGAS.cgas(
out_files,
sed_string=
"s/^Description:.*/Description: <unknown description>/",
output=outfile)
else:
CGAS.cat(out_files, output=outfile)
if tblout_outfile:
CGAS.cat(tblout_files, output=tblout_outfile)
if domtblout_outfile:
CGAS.cat(domtblout_files, output=domtblout_outfile)
if pfamtblout_outfile:
CGAS.cat(pfamtblout_files, output=pfamtblout_outfile)
if remove_tmp_dirs:
if splited_tblout_dir:
shutil.rmtree(splited_tblout_dir)
if splited_domtblout_dir:
shutil.rmtree(splited_domtblout_dir)
if splited_pfamtblout_dir:
shutil.rmtree(splited_pfamtblout_dir)
for tmp_dir in splited_dir, splited_out_dir:
shutil.rmtree(tmp_dir)
parser.add_argument("-d",
"--top_hits_dir",
action="store",
dest="top_hits_dir",
default="top_hits_dir/",
type=check_path,
help="Directory to write intermediate(splited) output")
parser.add_argument("-r",
"--retain_splited_output",
action="store_true",
dest="retain",
help="Retain splited output")
args = parser.parse_args()
save_mkdir(args.top_hits_dir)
def handle_input(filename):
sys.stdout.write("Handling %s\n" % filename)
not_significant_ids = IdList()
not_found_ids = IdList()
prefix = split_filename(filename)[1]
index_file = "%s.tmp.idx" % prefix
hmm_dict = SearchIO.index_db(index_file, filename, args.format)
if args.output == "stdout":
out_fd = sys.stdout
else:
out_fd = open("%s%s.top_hits" % (args.top_hits_dir, prefix), "w")
out_fd.write("#query\thit\tevalue\tbitscore\n")
def parallel_alignment(self,
query_file,
target_file,
model,
num_of_recs_per_file=None,
show_alignment=None,
show_sugar=True,
show_cigar=None,
show_vulgar=None,
show_query_gff=None,
show_target_gff=None,
store_intermediate_files=False,
splited_fasta_dir="splited_fasta_dir",
splited_result_dir="splited_output",
number_of_results_to_report=None,
other_options=None,
num_of_files=None,
converted_output_dir="converted_output"):
splited_filename = split_filename(query_file)
self.split_fasta(query_file,
splited_fasta_dir,
num_of_recs_per_file=num_of_recs_per_file,
num_of_files=num_of_files,
output_prefix=splited_filename[1])
common_options = self.parse_common_options(
model,
show_alignment=show_alignment,
show_sugar=show_sugar,
show_cigar=show_cigar,
show_vulgar=show_vulgar,
show_query_gff=show_query_gff,
show_target_gff=show_target_gff,
number_of_results_to_report=number_of_results_to_report,
other_options=other_options)
options_list = []
splited_files = os.listdir(splited_fasta_dir)
save_mkdir(splited_result_dir)
#save_mkdir(converted_output_dir)
for filename in splited_files:
filename_list = split_filename(filename)
options = common_options
options += " -q %s/%s" % (splited_fasta_dir, filename)
options += " -t %s" % target_file
options += " > %s/%s.output" % (splited_result_dir,
filename_list[1])
options_list.append(options)
self.parallel_execute(options_list)
"""
for filename in splited_files:
trf_output_file = "%s/%s.%i.%i.%i.%i.%i.%i.%i.dat" % (splited_result_dir, filename,
matching_weight, mismatching_penalty,
indel_penalty, match_probability,
indel_probability,
min_alignment_score, max_period)
self.convert_trf_report(trf_output_file, "%s/%s" % (converted_output_dir, filename))
for suffix in (".rep", ".gff", ".simple.gff", ".short.tab", ".wide.tab"):
file_str = ""
merged_file = "%s%s" % (output_prefix, suffix)
for filename in splited_files:
file_str += " %s/%s%s" % (converted_output_dir, filename, suffix)
CGAS.cat(file_str, merged_file)
"""
if not store_intermediate_files:
shutil.rmtree(splited_fasta_dir)
def parallel_search_tandem_repeat(self,
query_file,
output_prefix,
matching_weight=2,
mismatching_penalty=7,
indel_penalty=7,
match_probability=80,
indel_probability=10,
min_alignment_score=50,
max_period=500,
report_flanking_sequences=False,
splited_fasta_dir="splited_fasta_dir",
splited_result_dir="splited_output",
converted_output_dir="converted_output",
max_len_per_file=100000,
store_intermediate_files=False):
work_dir = os.getcwd()
splited_filename = split_filename(query_file)
self.split_fasta_by_seq_len(query_file,
splited_fasta_dir,
max_len_per_file=max_len_per_file,
output_prefix=splited_filename[1])
common_options = self.parse_common_options(
matching_weight=matching_weight,
mismatching_penalty=mismatching_penalty,
indel_penalty=indel_penalty,
match_probability=match_probability,
indel_probability=indel_probability,
min_alignment_score=min_alignment_score,
max_period=max_period,
report_flanking_sequences=report_flanking_sequences,
make_dat_file=True)
common_options += " -h" # suppress html output
options_list = []
splited_files = os.listdir(splited_fasta_dir)
save_mkdir(splited_result_dir)
save_mkdir(converted_output_dir)
os.chdir(splited_result_dir)
input_dir = splited_fasta_dir if (splited_fasta_dir[0] == "/") or (splited_fasta_dir[0] == "~") \
else "../%s" % splited_fasta_dir
for filename in splited_files:
file_options = "%s/%s" % (input_dir, filename)
file_options += common_options
options_list.append(file_options)
self.parallel_execute(options_list)
os.chdir(work_dir)
for filename in splited_files:
trf_output_file = "%s/%s.%i.%i.%i.%i.%i.%i.%i.dat" % (
splited_result_dir, filename, matching_weight,
mismatching_penalty, indel_penalty, match_probability,
indel_probability, min_alignment_score, max_period)
self.convert_trf_report(trf_output_file,
"%s/%s" % (converted_output_dir, filename))
for suffix in (".rep", ".gff", ".simple.gff", ".short.tab",
".wide.tab"):
file_str = ""
merged_file = "%s%s" % (output_prefix, suffix)
for filename in splited_files:
file_str += " %s/%s%s" % (converted_output_dir, filename,
suffix)
CGAS.cat(file_str, merged_file)
if not store_intermediate_files:
shutil.rmtree(splited_fasta_dir)
shutil.rmtree(splited_result_dir)
shutil.rmtree(converted_output_dir)
action="store",
dest="hmmer_dir",
default="",
help="Directory with hmmer v3.1 binaries")
args = parser.parse_args()
input_filename_list = split_filename(args.input)
input_filename = input_filename_list[1] + input_filename_list[2]
workdir_dir = "%s.transdecoder_dir/" % input_filename
pep_from_longest_orfs = "%s/longest_orfs.pep" % workdir_dir
hmmscan_dir = "hmmscan_vs_pfam/"
blastp_dir = "blastp_vs_uniref/"
save_mkdir(hmmscan_dir)
save_mkdir(blastp_dir)
hmmscan_splited_fasta_dir = "%ssplited_fasta_dir/" % hmmscan_dir
splited_domtblout_dir = "%ssplited_domtblout_dir/" % hmmscan_dir
hmmscan_vs_pfam_output = "%s%s.pfam.hits" % (hmmscan_dir, input_filename)
domtblout_outfile = "%s%s.pfam.domtblout" % (
hmmscan_dir, input_filename) if args.pfam_database else None
blastp_outfile = "%s%s.blastp.hits" % (
blastp_dir, input_filename) if args.blast_database else None
blastp_split_dir = "%ssplited_fasta_dir/" % blastp_dir
blastp_splited_output_dir = "%ssplited_output_dir" % blastp_dir
HMMER3.path = args.hmmer_dir
HMMER3.threads = args.threads
BLASTp.threads = args.threads
