def collapse_to_reference(hq_fq, hq_lq_prefix_pickle, out_dir, args, out_prefix="touse"): """ hq_fq --- HQ isoforms in fastq format. out_dir --- where to output results. min_count --- minimum # of supportive FLNC reads to call an isoform HQ First map HQ isoforms against reference ($gmap_db_dir/$gmap_db_name), next collapse HQ isoforms to representative isoforms based on mapping, and finally map representative isoforms to reference and return sorted SAM output. """ gmap_db, gmap_name = args.gmap_db, args.gmap_name # Map HQ isoforms to GMAP reference genome log.info("Mapping HQ isoforms to reference.") log.debug("HQ isoforms: %s", hq_fq) log.debug("reference: %s/%s", gmap_db, gmap_name) hq_sam = op.join(out_dir, "%s.sorted.sam" % op.basename(hq_fq)) map_isoforms_and_sort(input_filename=hq_fq, sam_filename=hq_sam, gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC) log.info( "Collapsing and filtering HQ isoforms to create representative isoforms." ) # Post mapping to genome analysis, including # * collapse polished HQ isoform clusters into groups # * count abundance of collapsed isoform groups # * filter collapsed isoforms based on abundance info rep_fq = op.join(out_dir, "%s.rep.fastq" % out_prefix) post_mapping_to_genome_runner(in_isoforms=hq_fq, in_sam=hq_sam, in_pickle=hq_lq_prefix_pickle, out_isoforms=rep_fq, out_gff=None, out_abundance=None, out_group=None, out_read_stat=None, min_aln_coverage=args.min_aln_coverage, min_aln_identity=args.min_aln_identity, min_flnc_coverage=args.min_flnc_coverage, max_fuzzy_junction=args.max_fuzzy_junction, allow_extra_5exon=args.allow_extra_5exon, min_count=args.min_count) rep_sam = op.join(out_dir, "%s.sorted.sam" % rep_fq) # Map representitive isoforms to reference map_isoforms_and_sort(input_filename=rep_fq, sam_filename=rep_sam, gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC) return rep_sam
def resolved_tool_contract_runner(rtc): """Run given a resolved tool contract""" gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(rtc.task.input_files[1]) map_isoforms_and_sort(input_filename=rtc.task.input_files[0], sam_filename=rtc.task.output_files[0], gmap_db_dir=gmap_db_dir, gmap_db_name=gmap_db_name, gmap_nproc=rtc.task.options[Constants.GMAP_NPROC_ID]) return 0
def resolved_tool_contract_runner(rtc): """Run given a resolved tool contract""" gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds( rtc.task.input_files[1]) map_isoforms_and_sort(input_filename=rtc.task.input_files[0], sam_filename=rtc.task.output_files[0], gmap_db_dir=gmap_db_dir, gmap_db_name=gmap_db_name, gmap_nproc=rtc.task.options[Constants.GMAP_NPROC_ID]) return 0
def args_runner(args): """Run given input args. e.g., map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_db=<path-to-db> --gmap_name=<name> map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_ds=<path-to-xml> """ gmap_db_dir, gmap_db_name = args.gmap_db, args.gmap_name if args.gmap_ds is not None: gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(args.gmap_ds) map_isoforms_and_sort(input_filename=args.input_filename, sam_filename=args.sam_filename, gmap_db_dir=gmap_db_dir, gmap_db_name=gmap_db_name, gmap_nproc=args.gmap_nproc) return 0
def args_runner(args): """Run given input args. e.g., map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_db=<path-to-db> --gmap_name=<name> map_isoforms_to_genome.py hq_isoforms.fastq out.sam --gmap_ds=<path-to-xml> """ gmap_db_dir, gmap_db_name = args.gmap_db, args.gmap_name if args.gmap_ds is not None: gmap_db_dir, gmap_db_name = gmap_db_and_name_from_ds(args.gmap_ds) map_isoforms_and_sort(input_filename=args.input_filename, sam_filename=args.sam_filename, gmap_db_dir=gmap_db_dir, gmap_db_name=gmap_db_name, gmap_nproc=args.gmap_nproc) return 0
def collapse_to_reference(hq_fq, hq_lq_prefix_pickle, out_dir, args, out_prefix="touse"): """ hq_fq --- HQ isoforms in fastq format. out_dir --- where to output results. min_count --- minimum # of supportive FLNC reads to call an isoform HQ First map HQ isoforms against reference ($gmap_db_dir/$gmap_db_name), next collapse HQ isoforms to representative isoforms based on mapping, and finally map representative isoforms to reference and return sorted SAM output. """ gmap_db, gmap_name = args.gmap_db, args.gmap_name # Map HQ isoforms to GMAP reference genome log.info("Mapping HQ isoforms to reference.") log.debug("HQ isoforms: %s", hq_fq) log.debug("reference: %s/%s", gmap_db, gmap_name) hq_sam = op.join(out_dir, "%s.sorted.sam" % op.basename(hq_fq)) map_isoforms_and_sort(input_filename=hq_fq, sam_filename=hq_sam, gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC) log.info("Collapsing and filtering HQ isoforms to create representative isoforms.") # Post mapping to genome analysis, including # * collapse polished HQ isoform clusters into groups # * count abundance of collapsed isoform groups # * filter collapsed isoforms based on abundance info rep_fq = op.join(out_dir, "%s.rep.fastq" % out_prefix) post_mapping_to_genome_runner(in_isoforms=hq_fq, in_sam=hq_sam, in_pickle=hq_lq_prefix_pickle, out_isoforms=rep_fq, out_gff=None, out_abundance=None, out_group=None, out_read_stat=None, min_aln_coverage=args.min_aln_coverage, min_aln_identity=args.min_aln_identity, min_flnc_coverage=args.min_flnc_coverage, max_fuzzy_junction=args.max_fuzzy_junction, allow_extra_5exon=args.allow_extra_5exon, min_count=args.min_count) rep_sam = op.join(out_dir, "%s.sorted.sam" % rep_fq) # Map representitive isoforms to reference map_isoforms_and_sort(input_filename=rep_fq, sam_filename=rep_sam, gmap_db_dir=gmap_db, gmap_db_name=gmap_name, gmap_nproc=GMAP_NPROC) return rep_sam
def test_map_isoforms_and_sort(self): """Test map_isoforms_and_sort""" out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fasta.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA, sam_filename=out_fn, gmap_db_dir=self.gmap_db_dir, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fastq.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ, sam_filename=out_fn, gmap_db_dir=self.gmap_db_dir, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fasta_ds.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA_DS, sam_filename=out_fn, gmap_db_dir=self.gmap_db_dir, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test map_isoforms_and_sort_fastq_ds.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ_DS, sam_filename=out_fn, gmap_db_dir=self.gmap_db_dir, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn))
def test_map_isoforms_and_sort(self): """Test map_isoforms_and_sort""" out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fasta.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA, sam_filename=out_fn, gmap_db_dir=GMAP_DB, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fastq.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ, sam_filename=out_fn, gmap_db_dir=GMAP_DB, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fasta_ds.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTA_DS, sam_filename=out_fn, gmap_db_dir=GMAP_DB, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn)) out_fn = op.join(_OUT_DIR_, 'test_map_isoforms_and_sort_fastq_ds.sam') rmpath(out_fn) map_isoforms_and_sort(input_filename=GMAP_INPUT_FASTQ_DS, sam_filename=out_fn, gmap_db_dir=GMAP_DB, gmap_db_name=GMAP_NAME, gmap_nproc=10) self.assertTrue(op.exists(out_fn))
def args_runner(args): """args runner""" logging.info("%s arguments are:\n%s\n", __file__, args) # sanity check arguments _sanity_check_args(args) # make option objects ice_opts = IceOptions(quiver=args.quiver, use_finer_qv=args.use_finer_qv, targeted_isoseq=args.targeted_isoseq, ece_penalty=args.ece_penalty, ece_min_len=args.ece_min_len, flnc_reads_per_split=args.flnc_reads_per_split, nfl_reads_per_split=args.nfl_reads_per_split) sge_opts = SgeOptions(unique_id=args.unique_id, use_sge=args.use_sge, max_sge_jobs=args.max_sge_jobs, blasr_nproc=args.blasr_nproc, quiver_nproc=args.quiver_nproc, gcon_nproc=args.gcon_nproc, sge_env_name=args.sge_env_name, sge_queue=args.sge_queue) ipq_opts = IceQuiverHQLQOptions( qv_trim_5=args.qv_trim_5, qv_trim_3=args.qv_trim_3, hq_quiver_min_accuracy=args.hq_quiver_min_accuracy) # (1) separate flnc reads into bins logging.info("Separating FLNC reads into bins.") tofu_f = TofuFiles(tofu_dir=args.tofu_dir) s = SeparateFLNCRunner(flnc_fa=args.flnc_fa, root_dir=args.tofu_dir, out_pickle=tofu_f.separate_flnc_pickle, bin_size_kb=args.bin_size_kb, bin_by_primer=args.bin_by_primer, bin_manual=args.bin_manual, max_base_limit_MB=args.max_base_limit_MB) s.run() flnc_files = SeparateFLNCBase.convert_pickle_to_sorted_flnc_files( tofu_f.separate_flnc_pickle) logging.info("Separated FLNC reads bins are %s", flnc_files) # (2) apply 'pbtranscript cluster' to each bin # run ICE/Quiver (the whole thing), providing the fasta_fofn logging.info("Running ICE/Polish on separated FLNC reads bins.") split_dirs = [] for flnc_file in flnc_files: split_dir = op.join(realpath(op.dirname(flnc_file)), "cluster_out") mkdir(split_dir) split_dirs.append(split_dir) cur_out_cons = op.join(split_dir, "consensus_isoforms.fasta") ipq_f = IceQuiverPostprocess(root_dir=split_dir, ipq_opts=ipq_opts) if op.exists(ipq_f.quivered_good_fq): logging.warning("HQ polished isoforms %s already exist. SKIP!", ipq_f.quivered_good_fq) continue else: logging.info("Running ICE/Quiver on %s", split_dir) rmpath(cur_out_cons) obj = Cluster(root_dir=split_dir, flnc_fa=flnc_file, nfl_fa=args.nfl_fa, bas_fofn=args.bas_fofn, ccs_fofn=args.ccs_fofn, fasta_fofn=args.fasta_fofn, out_fa=cur_out_cons, sge_opts=sge_opts, ice_opts=ice_opts, ipq_opts=ipq_opts) if args.mem_debug: # DEBUG from memory_profiler import memory_usage start_t = time.time() mem_usage = memory_usage(obj.run, interval=60) end_t = time.time() with open('mem_debug.log', 'a') as f: f.write("Running ICE/Quiver on {0} took {1} secs.\n".format( split_dir, end_t - start_t)) f.write("Maximum memory usage: {0}\n".format(max(mem_usage))) f.write("Memory usage: {0}\n".format(mem_usage)) else: obj.run() if not args.keep_tmp_files: # by deafult, delete all tempory files. logging.info("Deleting %s", ipq_f.tmp_dir) subprocess.Popen(['rm', '-rf', '%s' % ipq_f.tmp_dir]) logging.info("Deleting %s", ipq_f.quivered_dir) subprocess.Popen(['rm', '-rf', '%s' % ipq_f.quivered_dir]) # (3) merge polished isoform cluster from all bins logging.info("Merging isoforms from all bins to %s.", tofu_f.combined_dir) c = CombineRunner(combined_dir=tofu_f.combined_dir, sample_name=get_sample_name(args.sample_name), split_dirs=split_dirs, ipq_opts=ipq_opts) c.run() if args.summary_fn is not None: ln(tofu_f.all_cluster_summary_fn, args.summary_fn) if args.report_fn is not None: ln(tofu_f.all_cluster_report_fn, args.report_fn) # (4) map HQ isoforms to GMAP reference genome map_isoforms_and_sort(input_filename=tofu_f.all_hq_fq, sam_filename=tofu_f.sorted_gmap_sam, gmap_db_dir=args.gmap_db, gmap_db_name=args.gmap_name, gmap_nproc=args.gmap_nproc) # (5) post mapping to genome analysis, including # * collapse polished HQ isoform clusters into groups # * count abundance of collapsed isoform groups # * filter collapsed isoforms based on abundance info logging.info("Post mapping to genome analysis.") out_isoforms = args.collapsed_filtered_fn if any(out_isoforms.endswith(ext) for ext in (".fa", ".fasta")): in_isoforms = tofu_f.all_hq_fa elif any(out_isoforms.endswith(ext) for ext in (".fq", ".fastq")): in_isoforms = tofu_f.all_hq_fq else: raise ValueError("Output file %s must be FASTA or FASTQ!" % out_isoforms) post_mapping_to_genome_runner(in_isoforms=in_isoforms, in_sam=tofu_f.sorted_gmap_sam, in_pickle=tofu_f.hq_lq_prefix_dict_pickle, out_isoforms=args.collapsed_filtered_fn, out_gff=args.gff_fn, out_abundance=args.abundance_fn, out_group=args.group_fn, out_read_stat=args.read_stat_fn, min_aln_coverage=args.min_aln_coverage, min_aln_identity=args.min_aln_identity, min_flnc_coverage=args.min_flnc_coverage, max_fuzzy_junction=args.max_fuzzy_junction, allow_extra_5exon=args.allow_extra_5exon, min_count=args.min_count) return 0
def args_runner(args): """args runner""" logging.info("%s arguments are:\n%s\n", __file__, args) # sanity check arguments _sanity_check_args(args) # make option objects ice_opts = IceOptions(quiver=args.quiver, use_finer_qv=args.use_finer_qv, targeted_isoseq=args.targeted_isoseq, ece_penalty=args.ece_penalty, ece_min_len=args.ece_min_len, nfl_reads_per_split=args.nfl_reads_per_split) sge_opts = SgeOptions(unique_id=args.unique_id, use_sge=args.use_sge, max_sge_jobs=args.max_sge_jobs, blasr_nproc=args.blasr_nproc, quiver_nproc=args.quiver_nproc, gcon_nproc=args.gcon_nproc, sge_env_name=args.sge_env_name, sge_queue=args.sge_queue) ipq_opts = IceQuiverHQLQOptions(qv_trim_5=args.qv_trim_5, qv_trim_3=args.qv_trim_3, hq_quiver_min_accuracy=args.hq_quiver_min_accuracy) # (1) separate flnc reads into bins logging.info("Separating FLNC reads into bins.") tofu_f = TofuFiles(tofu_dir=args.tofu_dir) s = SeparateFLNCRunner(flnc_fa=args.flnc_fa, root_dir=args.tofu_dir, out_pickle=tofu_f.separate_flnc_pickle, bin_size_kb=args.bin_size_kb, bin_by_primer=args.bin_by_primer, bin_manual=args.bin_manual, max_base_limit_MB=args.max_base_limit_MB) s.run() flnc_files = SeparateFLNCBase.convert_pickle_to_sorted_flnc_files(tofu_f.separate_flnc_pickle) logging.info("Separated FLNC reads bins are %s", flnc_files) # (2) apply 'pbtranscript cluster' to each bin # run ICE/Quiver (the whole thing), providing the fasta_fofn logging.info("Running ICE/Polish on separated FLNC reads bins.") split_dirs = [] for flnc_file in flnc_files: split_dir = op.join(realpath(op.dirname(flnc_file)), "cluster_out") mkdir(split_dir) split_dirs.append(split_dir) cur_out_cons = op.join(split_dir, "consensus_isoforms.fasta") ipq_f = IceQuiverPostprocess(root_dir=split_dir, ipq_opts=ipq_opts) if op.exists(ipq_f.quivered_good_fq): logging.warning("HQ polished isoforms %s already exist. SKIP!", ipq_f.quivered_good_fq) continue else: logging.info("Running ICE/Quiver on %s", split_dir) rmpath(cur_out_cons) obj = Cluster(root_dir=split_dir, flnc_fa=flnc_file, nfl_fa=args.nfl_fa, bas_fofn=args.bas_fofn, ccs_fofn=args.ccs_fofn, fasta_fofn=args.fasta_fofn, out_fa=cur_out_cons, sge_opts=sge_opts, ice_opts=ice_opts, ipq_opts=ipq_opts) if args.mem_debug: # DEBUG from memory_profiler import memory_usage start_t = time.time() mem_usage = memory_usage(obj.run, interval=60) end_t = time.time() with open('mem_debug.log', 'a') as f: f.write("Running ICE/Quiver on {0} took {1} secs.\n".format(split_dir, end_t-start_t)) f.write("Maximum memory usage: {0}\n".format(max(mem_usage))) f.write("Memory usage: {0}\n".format(mem_usage)) else: obj.run() if not args.keep_tmp_files: # by deafult, delete all tempory files. logging.info("Deleting %s", ipq_f.tmp_dir) subprocess.Popen(['rm', '-rf', '%s' % ipq_f.tmp_dir]) logging.info("Deleting %s", ipq_f.quivered_dir) subprocess.Popen(['rm', '-rf', '%s' % ipq_f.quivered_dir]) # (3) merge polished isoform cluster from all bins logging.info("Merging isoforms from all bins to %s.", tofu_f.combined_dir) c = CombineRunner(combined_dir=tofu_f.combined_dir, sample_name=get_sample_name(args.sample_name), split_dirs=split_dirs, ipq_opts=ipq_opts) c.run() if args.summary_fn is not None: ln(tofu_f.all_cluster_summary_fn, args.summary_fn) if args.report_fn is not None: ln(tofu_f.all_cluster_report_fn, args.report_fn) # (4) map HQ isoforms to GMAP reference genome map_isoforms_and_sort(input_filename=tofu_f.all_hq_fq, sam_filename=tofu_f.sorted_gmap_sam, gmap_db_dir=args.gmap_db, gmap_db_name=args.gmap_name, gmap_nproc=args.gmap_nproc) # (5) post mapping to genome analysis, including # * collapse polished HQ isoform clusters into groups # * count abundance of collapsed isoform groups # * filter collapsed isoforms based on abundance info logging.info("Post mapping to genome analysis.") out_isoforms = args.collapsed_filtered_fn if any(out_isoforms.endswith(ext) for ext in (".fa", ".fasta")): in_isoforms = tofu_f.all_hq_fa elif any(out_isoforms.endswith(ext) for ext in (".fq", ".fastq")): in_isoforms = tofu_f.all_hq_fq else: raise ValueError("Output file %s must be FASTA or FASTQ!" % out_isoforms) post_mapping_to_genome_runner( in_isoforms=in_isoforms, in_sam=tofu_f.sorted_gmap_sam, in_pickle=tofu_f.hq_lq_prefix_dict_pickle, out_isoforms=args.collapsed_filtered_fn, out_gff=args.gff_fn, out_abundance=args.abundance_fn, out_group=args.group_fn, out_read_stat=args.read_stat_fn, min_aln_coverage=args.min_aln_coverage, min_aln_identity=args.min_aln_identity, min_flnc_coverage=args.min_flnc_coverage, max_fuzzy_junction=args.max_fuzzy_junction, allow_extra_5exon=args.allow_extra_5exon, min_count=args.min_count) return 0