def collapse_to_reference(hq_fq,
hq_lq_prefix_pickle,
out_dir,
args,
out_prefix="touse"):
"""
hq_fq --- HQ isoforms in fastq format.
out_dir --- where to output results.
min_count --- minimum # of supportive FLNC reads to call an isoform HQ
First map HQ isoforms against reference ($gmap_db_dir/$gmap_db_name),
next collapse HQ isoforms to representative isoforms based on mapping,
and finally map representative isoforms to reference and return sorted
SAM output.
"""
gmap_db, gmap_name = args.gmap_db, args.gmap_name
# Map HQ isoforms to GMAP reference genome
log.info("Mapping HQ isoforms to reference.")
log.debug("HQ isoforms: %s", hq_fq)
log.debug("reference: %s/%s", gmap_db, gmap_name)
hq_sam = op.join(out_dir, "%s.sorted.sam" % op.basename(hq_fq))
map_isoforms_and_sort(input_filename=hq_fq,
sam_filename=hq_sam,
gmap_db_dir=gmap_db,
gmap_db_name=gmap_name,
gmap_nproc=GMAP_NPROC)
log.info(
"Collapsing and filtering HQ isoforms to create representative isoforms."
)
# Post mapping to genome analysis, including
# * collapse polished HQ isoform clusters into groups
# * count abundance of collapsed isoform groups
# * filter collapsed isoforms based on abundance info
rep_fq = op.join(out_dir, "%s.rep.fastq" % out_prefix)
post_mapping_to_genome_runner(in_isoforms=hq_fq,
in_sam=hq_sam,
in_pickle=hq_lq_prefix_pickle,
out_isoforms=rep_fq,
out_gff=None,
out_abundance=None,
out_group=None,
out_read_stat=None,
min_aln_coverage=args.min_aln_coverage,
min_aln_identity=args.min_aln_identity,
min_flnc_coverage=args.min_flnc_coverage,
max_fuzzy_junction=args.max_fuzzy_junction,
allow_extra_5exon=args.allow_extra_5exon,
min_count=args.min_count)
rep_sam = op.join(out_dir, "%s.sorted.sam" % rep_fq)
# Map representitive isoforms to reference
map_isoforms_and_sort(input_filename=rep_fq,
sam_filename=rep_sam,
gmap_db_dir=gmap_db,
gmap_db_name=gmap_name,
gmap_nproc=GMAP_NPROC)
return rep_sam
def resolved_tool_contract_runner(rtc):
"""Run given a resolved tool contract"""
gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(rtc.task.input_files[1])
map_isoforms_and_sort(input_filename=rtc.task.input_files[0],
sam_filename=rtc.task.output_files[0],
gmap_db_dir=gmap_db_dir,
gmap_db_name=gmap_db_name,
gmap_nproc=rtc.task.options[Constants.GMAP_NPROC_ID])
return 0
def resolved_tool_contract_runner(rtc):
"""Run given a resolved tool contract"""
gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(
rtc.task.input_files[1])
map_isoforms_and_sort(input_filename=rtc.task.input_files[0],
sam_filename=rtc.task.output_files[0],
gmap_db_dir=gmap_db_dir,
gmap_db_name=gmap_db_name,
gmap_nproc=rtc.task.options[Constants.GMAP_NPROC_ID])
return 0
def args_runner(args):
"""Run given input args.
e.g.,
map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_db=<path-to-db> --gmap_name=<name>
map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_ds=<path-to-xml>
"""
gmap_db_dir, gmap_db_name = args.gmap_db, args.gmap_name
if args.gmap_ds is not None:
gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(args.gmap_ds)
map_isoforms_and_sort(input_filename=args.input_filename,
sam_filename=args.sam_filename,
gmap_db_dir=gmap_db_dir,
gmap_db_name=gmap_db_name,
gmap_nproc=args.gmap_nproc)
return 0
def args_runner(args):
"""Run given input args.
e.g.,
map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_db=<path-to-db> --gmap_name=<name>
map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_ds=<path-to-xml>
"""
gmap_db_dir, gmap_db_name = args.gmap_db, args.gmap_name
if args.gmap_ds is not None:
gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(args.gmap_ds)
map_isoforms_and_sort(input_filename=args.input_filename,
sam_filename=args.sam_filename,
gmap_db_dir=gmap_db_dir,
gmap_db_name=gmap_db_name,
gmap_nproc=args.gmap_nproc)
return 0
def collapse_to_reference(hq_fq, hq_lq_prefix_pickle, out_dir, args, out_prefix="touse"):
"""
hq_fq --- HQ isoforms in fastq format.
out_dir --- where to output results.
min_count --- minimum # of supportive FLNC reads to call an isoform HQ
First map HQ isoforms against reference ($gmap_db_dir/$gmap_db_name),
next collapse HQ isoforms to representative isoforms based on mapping,
and finally map representative isoforms to reference and return sorted
SAM output.
"""
gmap_db, gmap_name = args.gmap_db, args.gmap_name
# Map HQ isoforms to GMAP reference genome
log.info("Mapping HQ isoforms to reference.")
log.debug("HQ isoforms: %s", hq_fq)
log.debug("reference: %s/%s", gmap_db, gmap_name)
hq_sam = op.join(out_dir, "%s.sorted.sam" % op.basename(hq_fq))
map_isoforms_and_sort(input_filename=hq_fq, sam_filename=hq_sam,
gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC)
log.info("Collapsing and filtering HQ isoforms to create representative isoforms.")
# Post mapping to genome analysis, including
# * collapse polished HQ isoform clusters into groups
# * count abundance of collapsed isoform groups
# * filter collapsed isoforms based on abundance info
rep_fq = op.join(out_dir, "%s.rep.fastq" % out_prefix)
post_mapping_to_genome_runner(in_isoforms=hq_fq, in_sam=hq_sam,
in_pickle=hq_lq_prefix_pickle,
out_isoforms=rep_fq, out_gff=None, out_abundance=None,
out_group=None, out_read_stat=None,
min_aln_coverage=args.min_aln_coverage,
min_aln_identity=args.min_aln_identity,
min_flnc_coverage=args.min_flnc_coverage,
max_fuzzy_junction=args.max_fuzzy_junction,
allow_extra_5exon=args.allow_extra_5exon,
min_count=args.min_count)
rep_sam = op.join(out_dir, "%s.sorted.sam" % rep_fq)
# Map representitive isoforms to reference
map_isoforms_and_sort(input_filename=rep_fq, sam_filename=rep_sam,
gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC)
return rep_sam
def test_map_isoforms_and_sort(self):
"""Test map_isoforms_and_sort"""
out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fasta.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA,
sam_filename=out_fn,
gmap_db_dir=self.gmap_db_dir,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fastq.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ,
sam_filename=out_fn,
gmap_db_dir=self.gmap_db_dir,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fasta_ds.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA_DS,
sam_filename=out_fn,
gmap_db_dir=self.gmap_db_dir,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fastq_ds.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ_DS,
sam_filename=out_fn,
gmap_db_dir=self.gmap_db_dir,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
def test_map_isoforms_and_sort(self):
"""Test map_isoforms_and_sort"""
out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fasta.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA,
sam_filename=out_fn,
gmap_db_dir=GMAP_DB,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fastq.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ,
sam_filename=out_fn,
gmap_db_dir=GMAP_DB,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fasta_ds.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA_DS,
sam_filename=out_fn,
gmap_db_dir=GMAP_DB,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fastq_ds.sam')
rmpath(out_fn)
map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ_DS,
sam_filename=out_fn,
gmap_db_dir=GMAP_DB,
gmap_db_name=GMAP_NAME,
gmap_nproc=10)
self.assertTrue(op.exists(out_fn))
def args_runner(args):
"""args runner"""
logging.info("%s arguments are:\n%s\n", __file__, args)
# sanity check arguments
_sanity_check_args(args)
# make option objects
ice_opts = IceOptions(quiver=args.quiver,
use_finer_qv=args.use_finer_qv,
targeted_isoseq=args.targeted_isoseq,
ece_penalty=args.ece_penalty,
ece_min_len=args.ece_min_len,
flnc_reads_per_split=args.flnc_reads_per_split,
nfl_reads_per_split=args.nfl_reads_per_split)
sge_opts = SgeOptions(unique_id=args.unique_id,
use_sge=args.use_sge,
max_sge_jobs=args.max_sge_jobs,
blasr_nproc=args.blasr_nproc,
quiver_nproc=args.quiver_nproc,
gcon_nproc=args.gcon_nproc,
sge_env_name=args.sge_env_name,
sge_queue=args.sge_queue)
ipq_opts = IceQuiverHQLQOptions(
qv_trim_5=args.qv_trim_5,
qv_trim_3=args.qv_trim_3,
hq_quiver_min_accuracy=args.hq_quiver_min_accuracy)
# (1) separate flnc reads into bins
logging.info("Separating FLNC reads into bins.")
tofu_f = TofuFiles(tofu_dir=args.tofu_dir)
s = SeparateFLNCRunner(flnc_fa=args.flnc_fa,
root_dir=args.tofu_dir,
out_pickle=tofu_f.separate_flnc_pickle,
bin_size_kb=args.bin_size_kb,
bin_by_primer=args.bin_by_primer,
bin_manual=args.bin_manual,
max_base_limit_MB=args.max_base_limit_MB)
s.run()
flnc_files = SeparateFLNCBase.convert_pickle_to_sorted_flnc_files(
tofu_f.separate_flnc_pickle)
logging.info("Separated FLNC reads bins are %s", flnc_files)
# (2) apply 'pbtranscript cluster' to each bin
# run ICE/Quiver (the whole thing), providing the fasta_fofn
logging.info("Running ICE/Polish on separated FLNC reads bins.")
split_dirs = []
for flnc_file in flnc_files:
split_dir = op.join(realpath(op.dirname(flnc_file)), "cluster_out")
mkdir(split_dir)
split_dirs.append(split_dir)
cur_out_cons = op.join(split_dir, "consensus_isoforms.fasta")
ipq_f = IceQuiverPostprocess(root_dir=split_dir, ipq_opts=ipq_opts)
if op.exists(ipq_f.quivered_good_fq):
logging.warning("HQ polished isoforms %s already exist. SKIP!",
ipq_f.quivered_good_fq)
continue
else:
logging.info("Running ICE/Quiver on %s", split_dir)
rmpath(cur_out_cons)
obj = Cluster(root_dir=split_dir,
flnc_fa=flnc_file,
nfl_fa=args.nfl_fa,
bas_fofn=args.bas_fofn,
ccs_fofn=args.ccs_fofn,
fasta_fofn=args.fasta_fofn,
out_fa=cur_out_cons,
sge_opts=sge_opts,
ice_opts=ice_opts,
ipq_opts=ipq_opts)
if args.mem_debug: # DEBUG
from memory_profiler import memory_usage
start_t = time.time()
mem_usage = memory_usage(obj.run, interval=60)
end_t = time.time()
with open('mem_debug.log', 'a') as f:
f.write("Running ICE/Quiver on {0} took {1} secs.\n".format(
split_dir, end_t - start_t))
f.write("Maximum memory usage: {0}\n".format(max(mem_usage)))
f.write("Memory usage: {0}\n".format(mem_usage))
else:
obj.run()
if not args.keep_tmp_files: # by deafult, delete all tempory files.
logging.info("Deleting %s", ipq_f.tmp_dir)
subprocess.Popen(['rm', '-rf', '%s' % ipq_f.tmp_dir])
logging.info("Deleting %s", ipq_f.quivered_dir)
subprocess.Popen(['rm', '-rf', '%s' % ipq_f.quivered_dir])
# (3) merge polished isoform cluster from all bins
logging.info("Merging isoforms from all bins to %s.", tofu_f.combined_dir)
c = CombineRunner(combined_dir=tofu_f.combined_dir,
sample_name=get_sample_name(args.sample_name),
split_dirs=split_dirs,
ipq_opts=ipq_opts)
c.run()
if args.summary_fn is not None:
ln(tofu_f.all_cluster_summary_fn, args.summary_fn)
if args.report_fn is not None:
ln(tofu_f.all_cluster_report_fn, args.report_fn)
# (4) map HQ isoforms to GMAP reference genome
map_isoforms_and_sort(input_filename=tofu_f.all_hq_fq,
sam_filename=tofu_f.sorted_gmap_sam,
gmap_db_dir=args.gmap_db,
gmap_db_name=args.gmap_name,
gmap_nproc=args.gmap_nproc)
# (5) post mapping to genome analysis, including
# * collapse polished HQ isoform clusters into groups
# * count abundance of collapsed isoform groups
# * filter collapsed isoforms based on abundance info
logging.info("Post mapping to genome analysis.")
out_isoforms = args.collapsed_filtered_fn
if any(out_isoforms.endswith(ext) for ext in (".fa", ".fasta")):
in_isoforms = tofu_f.all_hq_fa
elif any(out_isoforms.endswith(ext) for ext in (".fq", ".fastq")):
in_isoforms = tofu_f.all_hq_fq
else:
raise ValueError("Output file %s must be FASTA or FASTQ!" %
out_isoforms)
post_mapping_to_genome_runner(in_isoforms=in_isoforms,
in_sam=tofu_f.sorted_gmap_sam,
in_pickle=tofu_f.hq_lq_prefix_dict_pickle,
out_isoforms=args.collapsed_filtered_fn,
out_gff=args.gff_fn,
out_abundance=args.abundance_fn,
out_group=args.group_fn,
out_read_stat=args.read_stat_fn,
min_aln_coverage=args.min_aln_coverage,
min_aln_identity=args.min_aln_identity,
min_flnc_coverage=args.min_flnc_coverage,
max_fuzzy_junction=args.max_fuzzy_junction,
allow_extra_5exon=args.allow_extra_5exon,
min_count=args.min_count)
return 0
def args_runner(args):
"""args runner"""
logging.info("%s arguments are:\n%s\n", __file__, args)
# sanity check arguments
_sanity_check_args(args)
# make option objects
ice_opts = IceOptions(quiver=args.quiver, use_finer_qv=args.use_finer_qv,
targeted_isoseq=args.targeted_isoseq,
ece_penalty=args.ece_penalty, ece_min_len=args.ece_min_len,
nfl_reads_per_split=args.nfl_reads_per_split)
sge_opts = SgeOptions(unique_id=args.unique_id, use_sge=args.use_sge,
max_sge_jobs=args.max_sge_jobs, blasr_nproc=args.blasr_nproc,
quiver_nproc=args.quiver_nproc, gcon_nproc=args.gcon_nproc,
sge_env_name=args.sge_env_name, sge_queue=args.sge_queue)
ipq_opts = IceQuiverHQLQOptions(qv_trim_5=args.qv_trim_5, qv_trim_3=args.qv_trim_3,
hq_quiver_min_accuracy=args.hq_quiver_min_accuracy)
# (1) separate flnc reads into bins
logging.info("Separating FLNC reads into bins.")
tofu_f = TofuFiles(tofu_dir=args.tofu_dir)
s = SeparateFLNCRunner(flnc_fa=args.flnc_fa, root_dir=args.tofu_dir,
out_pickle=tofu_f.separate_flnc_pickle,
bin_size_kb=args.bin_size_kb, bin_by_primer=args.bin_by_primer,
bin_manual=args.bin_manual, max_base_limit_MB=args.max_base_limit_MB)
s.run()
flnc_files = SeparateFLNCBase.convert_pickle_to_sorted_flnc_files(tofu_f.separate_flnc_pickle)
logging.info("Separated FLNC reads bins are %s", flnc_files)
# (2) apply 'pbtranscript cluster' to each bin
# run ICE/Quiver (the whole thing), providing the fasta_fofn
logging.info("Running ICE/Polish on separated FLNC reads bins.")
split_dirs = []
for flnc_file in flnc_files:
split_dir = op.join(realpath(op.dirname(flnc_file)), "cluster_out")
mkdir(split_dir)
split_dirs.append(split_dir)
cur_out_cons = op.join(split_dir, "consensus_isoforms.fasta")
ipq_f = IceQuiverPostprocess(root_dir=split_dir, ipq_opts=ipq_opts)
if op.exists(ipq_f.quivered_good_fq):
logging.warning("HQ polished isoforms %s already exist. SKIP!", ipq_f.quivered_good_fq)
continue
else:
logging.info("Running ICE/Quiver on %s", split_dir)
rmpath(cur_out_cons)
obj = Cluster(root_dir=split_dir, flnc_fa=flnc_file,
nfl_fa=args.nfl_fa,
bas_fofn=args.bas_fofn,
ccs_fofn=args.ccs_fofn,
fasta_fofn=args.fasta_fofn,
out_fa=cur_out_cons, sge_opts=sge_opts,
ice_opts=ice_opts, ipq_opts=ipq_opts)
if args.mem_debug: # DEBUG
from memory_profiler import memory_usage
start_t = time.time()
mem_usage = memory_usage(obj.run, interval=60)
end_t = time.time()
with open('mem_debug.log', 'a') as f:
f.write("Running ICE/Quiver on {0} took {1} secs.\n".format(split_dir,
end_t-start_t))
f.write("Maximum memory usage: {0}\n".format(max(mem_usage)))
f.write("Memory usage: {0}\n".format(mem_usage))
else:
obj.run()
if not args.keep_tmp_files: # by deafult, delete all tempory files.
logging.info("Deleting %s", ipq_f.tmp_dir)
subprocess.Popen(['rm', '-rf', '%s' % ipq_f.tmp_dir])
logging.info("Deleting %s", ipq_f.quivered_dir)
subprocess.Popen(['rm', '-rf', '%s' % ipq_f.quivered_dir])
# (3) merge polished isoform cluster from all bins
logging.info("Merging isoforms from all bins to %s.", tofu_f.combined_dir)
c = CombineRunner(combined_dir=tofu_f.combined_dir,
sample_name=get_sample_name(args.sample_name),
split_dirs=split_dirs, ipq_opts=ipq_opts)
c.run()
if args.summary_fn is not None:
ln(tofu_f.all_cluster_summary_fn, args.summary_fn)
if args.report_fn is not None:
ln(tofu_f.all_cluster_report_fn, args.report_fn)
# (4) map HQ isoforms to GMAP reference genome
map_isoforms_and_sort(input_filename=tofu_f.all_hq_fq, sam_filename=tofu_f.sorted_gmap_sam,
gmap_db_dir=args.gmap_db, gmap_db_name=args.gmap_name,
gmap_nproc=args.gmap_nproc)
# (5) post mapping to genome analysis, including
# * collapse polished HQ isoform clusters into groups
# * count abundance of collapsed isoform groups
# * filter collapsed isoforms based on abundance info
logging.info("Post mapping to genome analysis.")
out_isoforms = args.collapsed_filtered_fn
if any(out_isoforms.endswith(ext) for ext in (".fa", ".fasta")):
in_isoforms = tofu_f.all_hq_fa
elif any(out_isoforms.endswith(ext) for ext in (".fq", ".fastq")):
in_isoforms = tofu_f.all_hq_fq
else:
raise ValueError("Output file %s must be FASTA or FASTQ!" % out_isoforms)
post_mapping_to_genome_runner(
in_isoforms=in_isoforms, in_sam=tofu_f.sorted_gmap_sam,
in_pickle=tofu_f.hq_lq_prefix_dict_pickle, out_isoforms=args.collapsed_filtered_fn,
out_gff=args.gff_fn, out_abundance=args.abundance_fn,
out_group=args.group_fn, out_read_stat=args.read_stat_fn,
min_aln_coverage=args.min_aln_coverage, min_aln_identity=args.min_aln_identity,
min_flnc_coverage=args.min_flnc_coverage, max_fuzzy_junction=args.max_fuzzy_junction,
allow_extra_5exon=args.allow_extra_5exon, min_count=args.min_count)
return 0
