def write_summary(self, summary_fn, isoforms_fa, hq_fa=None, lq_fa=None): """Extract number of consensus isoforms predicted, and total number of bases in all consensuus isoforms from isoforms_fa and write the two attributes to summary_fn. if hq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters if lq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters """ self.add_log("Writing a summary to {f}".format(f=summary_fn), level=logging.INFO) try: summary = ClusterSummary() with FastaReader(isoforms_fa) as reader: for r in reader: summary.numConsensusIsoforms += 1 summary.numTotalBases += len(r.sequence) if hq_fa is not None: summary.num_polished_hq_isoforms = 0 with FastaReader(hq_fa) as reader: for r in reader: summary.num_polished_hq_isoforms += 1 if lq_fa is not None: summary.num_polished_lq_isoforms = 0 with FastaReader(lq_fa) as reader: for r in reader: summary.num_polished_lq_isoforms += 1 summary.write(summary_fn) except ZeroDivisionError: errMsg = "No consensus isoforms predicted." self.add_log(errMsg, level=logging.ERROR) raise RuntimeError(errMsg)
def test_write(self): """Test ClusterSummary.write.""" outFN = op.join(self.testDir, "out/test_ClusterSummary.txt") stdoutFN = op.join(self.testDir, "stdout/test_ClusterSummary.txt") obj = ClusterSummary() obj.numConsensusIsoforms = 97 obj.numTotalBases = 97 * 3945 obj.write(outFN) self.assertTrue(filecmp.cmp(outFN, stdoutFN))
def __init__(self, root_dir, flnc_fa, nfl_fa, bas_fofn, ccs_fofn, out_fa, sge_opts, ice_opts, hq_isoforms_fa=None, hq_isoforms_fq=None, lq_isoforms_fa=None, lq_isoforms_fq=None, report_fn=None, summary_fn=None): super(Cluster, self).__init__(prog_name="Cluster", root_dir=root_dir, bas_fofn=bas_fofn, ccs_fofn=ccs_fofn) self.flnc_fa, self.nfl_fa, self.ccs_fofn = self._validateInputs( flnc_fa, nfl_fa, ccs_fofn) self.root_dir, self.out_fa = self._validateOutputs(root_dir, out_fa) self.hq_isoforms_fa = hq_isoforms_fa self.hq_isoforms_fq = hq_isoforms_fq self.lq_isoforms_fa = lq_isoforms_fa self.lq_isoforms_fq = lq_isoforms_fq self.sge_opts = sge_opts # SGE, CPU options and etc self.ice_opts = ice_opts # The ICE clutering algorithm options self.sanity_check() self._probqv = None # probability & quality value self._flnc_splitted_fas = [] # split flnc_fa into smaller files. self._nflncSplittedFas = [] # split nfl_fa into smaller files. self._logConfigs() # Log configurations self.iceinit = None self.icec = None self.iceq = None self.pol = None self.report_fn = realpath(report_fn) if report_fn is not None \ else op.join(self.root_dir, "cluster_report.csv") self.summary_fn = realpath(summary_fn) if summary_fn is not None \ else op.join(self.root_dir, "cluster_summary.txt") self.summary = ClusterSummary() self.add_log("Finishing create Cluster Object.", level=logging.INFO)
def __init__(self, root_dir, flnc_fa, nfl_fa, bas_fofn, ccs_fofn, out_fa, sge_opts, ice_opts, hq_isoforms_fa=None, hq_isoforms_fq=None, lq_isoforms_fa=None, lq_isoforms_fq=None, report_fn=None, summary_fn=None): super(Cluster, self).__init__(prog_name="Cluster", root_dir=root_dir, bas_fofn=bas_fofn, ccs_fofn=ccs_fofn) self.flnc_fa, self.nfl_fa, self.ccs_fofn = self._validateInputs( flnc_fa, nfl_fa, ccs_fofn) self.root_dir, self.out_fa = self._validateOutputs( root_dir, out_fa) self.hq_isoforms_fa = hq_isoforms_fa self.hq_isoforms_fq = hq_isoforms_fq self.lq_isoforms_fa = lq_isoforms_fa self.lq_isoforms_fq = lq_isoforms_fq self.sge_opts = sge_opts # SGE, CPU options and etc self.ice_opts = ice_opts # The ICE clutering algorithm options self.sanity_check() self._probqv = None # probability & quality value self._flnc_splitted_fas = [] # split flnc_fa into smaller files. self._nflncSplittedFas = [] # split nfl_fa into smaller files. self._logConfigs() # Log configurations self.iceinit = None self.icec = None self.iceq = None self.pol = None self.report_fn = realpath(report_fn) if report_fn is not None \ else op.join(self.root_dir, "cluster_report.csv") self.summary_fn = realpath(summary_fn) if summary_fn is not None \ else op.join(self.root_dir, "cluster_summary.txt") self.summary = ClusterSummary() self.add_log("Finishing create Cluster Object.", level=logging.INFO)
class Cluster(IceFiles): """ An object of `Cluster` calls the ICE algorithm to generate consensus isoforms. """ def __init__(self, root_dir, flnc_fa, nfl_fa, bas_fofn, ccs_fofn, out_fa, sge_opts, ice_opts, hq_isoforms_fa=None, hq_isoforms_fq=None, lq_isoforms_fa=None, lq_isoforms_fq=None, report_fn=None, summary_fn=None): super(Cluster, self).__init__(prog_name="Cluster", root_dir=root_dir, bas_fofn=bas_fofn, ccs_fofn=ccs_fofn) self.flnc_fa, self.nfl_fa, self.ccs_fofn = self._validateInputs( flnc_fa, nfl_fa, ccs_fofn) self.root_dir, self.out_fa = self._validateOutputs( root_dir, out_fa) self.hq_isoforms_fa = hq_isoforms_fa self.hq_isoforms_fq = hq_isoforms_fq self.lq_isoforms_fa = lq_isoforms_fa self.lq_isoforms_fq = lq_isoforms_fq self.sge_opts = sge_opts # SGE, CPU options and etc self.ice_opts = ice_opts # The ICE clutering algorithm options self.sanity_check() self._probqv = None # probability & quality value self._flnc_splitted_fas = [] # split flnc_fa into smaller files. self._nflncSplittedFas = [] # split nfl_fa into smaller files. self._logConfigs() # Log configurations self.iceinit = None self.icec = None self.iceq = None self.pol = None self.report_fn = realpath(report_fn) if report_fn is not None \ else op.join(self.root_dir, "cluster_report.csv") self.summary_fn = realpath(summary_fn) if summary_fn is not None \ else op.join(self.root_dir, "cluster_summary.txt") self.summary = ClusterSummary() self.add_log("Finishing create Cluster Object.", level=logging.INFO) def _validateInputs(self, _flnc_fa, _nfl_fa, _ccs_fofn): """Validate input files and return absolute expaneded paths.""" flnc_fa, nfl_fa, ccs_fofn = _flnc_fa, _nfl_fa, _ccs_fofn self.add_log("Checking input files.", level=logging.INFO) if flnc_fa is None or nfl_fa is None: raise ClusterException( "Input non-chimeric reads files needs to be specified.") else: flnc_fa, nfl_fa = realpath(flnc_fa), realpath(nfl_fa) if not op.exists(flnc_fa): raise ClusterException("Unable to find full-length " + "non-chimeric reads: {fn}".format(fn=flnc_fa)) if not op.exists(nfl_fa): raise ClusterException("Unable to find non-full-length " + "non-chimeric reads: {fn}".format(fn=nfl_fa)) if ccs_fofn is not None and not op.exists(ccs_fofn): raise ClusterException("Unable to find FOFN file: " + "{fn}".format(fn=ccs_fofn)) return (flnc_fa, nfl_fa, ccs_fofn) def _validateOutputs(self, _root_dir, _out_fa): """Validate outputs, create root_dir if it does not exist.""" self.add_log("Checking outputs.", level=logging.INFO) root_dir, out_fa = _root_dir, _out_fa if root_dir is None: self.add_log("Output directory needs to be specified.", level=logging.ERROR) if out_fa is None: self.add_log("Output consensus fasta needs to be specified.", level=logging.ERROR) root_dir = realpath(root_dir) out_fa = realpath(out_fa) if op.exists(root_dir): self.add_log("Output directory {d} already exists.". format(d=root_dir)) else: self.add_log("Creating output directory {d}.".format(d=root_dir)) os.mkdir(root_dir) if op.exists(out_fa): raise ClusterException("Consensus FASTA file {f} already exists.". format(f=out_fa)) return root_dir, out_fa def sanity_check(self): """Do sanity check before stat to run.""" errMsg = "" if self.ice_opts.quiver is True: if self.bas_fofn is None: errMsg = "A fofn of bas/bax.h5 files, e.g., input.fofn, " + \ "is required in order to polish consensus " + \ "isoforms using quiver." if self.nfl_fa is None: errMsg = "Non-full-length reads are required for polishing " + \ "consensus isoforms using quiver." if errMsg != "": self.add_log(errMsg, level=logging.ERROR) raise ValueError(errMsg) @property def configFN(self): """Return configuration file of the current run.""" return op.join(self.root_dir, "run_ice_config.txt") def _logConfigs(self): """Log configuration.""" with open (self.configFN, 'w', 0) as f: f.write('pbtranscript ' + get_version() + "\n") f.write(str(self.ice_opts) + "\n") f.write(str(self.sge_opts) + "\n") @property def initPickleFN(self): """Return path to pickle file with initial clusters.""" return op.join(self.root_dir, "init.uc.pickle") def _setProbQV(self, ccs_fofn=None, firstSplitFa=None): """Set probability and quality values. If a fofn file is specified, load QV from it. Otherwise, use a pre-defined probability model.""" if ccs_fofn is None: self.add_log("Loading predefined probabilities model.") self._probqv = ProbFromModel(0.01, 0.07, 0.06) else: self.add_log("Loading probabilities and QV from {f}.". format(f=firstSplitFa)) self._probqv = ProbFromQV(ccs_fofn, firstSplitFa) def writeSummary(self, fa, summary_fn, hq_fa=None, lq_fa=None): """Extract number of consensus isoforms predicted, and total number of bases in all consensuus isoforms from fa and write the two attributes to summary_fn. if hq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters if lq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters """ try: with FastaReader(fa) as reader: for r in reader: self.summary.numConsensusIsoforms += 1 self.summary.numTotalBases += len(r.sequence) if hq_fa is not None: self.summary.num_polished_hq_isoforms = 0 with FastaReader(hq_fa) as reader: for r in reader: self.summary.num_polished_hq_isoforms += 1 if lq_fa is not None: self.summary.num_polished_lq_isoforms = 0 with FastaReader(lq_fa) as reader: for r in reader: self.summary.num_polished_lq_isoforms += 1 self.summary.write(summary_fn) except ZeroDivisionError: errMsg = "No consensus isoforms predicted." self.add_log(errMsg, level=logging.ERROR) raise ClusterException(errMsg) def run(self): """Call ICE to cluster consensus isoforms.""" self.add_log("Start to run cluster.", level=logging.INFO) # Split flnc_fa into smaller files and save files to _flnc_splitted_fas. self.add_log("Splitting {flnc} into ".format(flnc=self.flnc_fa) + "smaller files each containing {n} reads.".format( n=self.ice_opts.flnc_reads_per_split), level=logging.INFO) self._flnc_splitted_fas = splitFasta( input_fasta=self.flnc_fa, reads_per_split=self.ice_opts.flnc_reads_per_split, out_dir=self.root_dir, out_prefix="input.split") self.add_log("Splitted files are: " + "\n".join(self._flnc_splitted_fas), level=logging.INFO) firstSplit = self._flnc_splitted_fas[0] # Set up probabbility and quality value model self._setProbQV(ccs_fofn=self.ccs_fofn, firstSplitFa=firstSplit) # Initialize cluster by clique # check if init.pickle already exists, if so, no need to run IceInit if os.path.exists(self.initPickleFN): self.add_log("{0} already exists. Reading to get uc.".format(self.initPickleFN), level=logging.INFO) with open(self.initPickleFN) as f: uc = cPickle.load(f) else: self.add_log("Finding maximal cliques.", level=logging.INFO) self.iceinit = IceInit(readsFa=firstSplit, qver_get_func=self._probqv.get_smoothed, ice_opts=self.ice_opts, sge_opts=self.sge_opts) uc = self.iceinit.uc # Dump uc to a file self.add_log("Dumping initial clusters to {f}".format( f=self.initPickleFN), level=logging.INFO) with open(self.initPickleFN, 'w') as f: cPickle.dump(uc, f) # Run IceIterative. self.add_log("Iteratively clustering.", level=logging.INFO) self.icec = IceIterative( fasta_filename=firstSplit, fasta_filenames_to_add=self._flnc_splitted_fas[1:], all_fasta_filename=self.flnc_fa, ccs_fofn=self.ccs_fofn, root_dir=self.root_dir, ice_opts=self.ice_opts, sge_opts=self.sge_opts, uc=uc, probQV=self._probqv) self.icec.run() clean_up_after_ICE(self.root_dir) # IceIterative done, write predicted (unplished) consensus isoforms # to an output fasta self.add_log("Creating a link to unpolished consensus isoforms.") ln(self.icec.final_consensus_fa, self.out_fa) # Call quiver to polish predicted consensus isoforms. if self.ice_opts.quiver is not True: self.add_log("Creating a link to cluster report.") ln(src=self.icec.report_fn, dst=self.report_fn) self.add_log("Writing a summary to {f}".format(f=self.summary_fn), level=logging.INFO) self.writeSummary(fa=self.out_fa, summary_fn=self.summary_fn) else: # self.ice_opts.quiver is True #TODO review code self.pol = Polish(root_dir=self.root_dir, nfl_fa=self.nfl_fa, bas_fofn=self.bas_fofn, ccs_fofn=self.ccs_fofn, hq_isoforms_fa=self.hq_isoforms_fa, hq_isoforms_fq=self.hq_isoforms_fq, lq_isoforms_fa=self.lq_isoforms_fa, lq_isoforms_fq=self.lq_isoforms_fq, ice_opts=self.ice_opts, sge_opts=self.sge_opts) self.pol.run() # cluster report self.add_log("Creating a link to cluster report.") ln(src=self.pol.iceq.report_fn, dst=self.report_fn) # Write a summary. self.add_log("Writing a summary to {f}".format(f=self.summary_fn), level=logging.INFO) self.writeSummary(fa=self.out_fa, summary_fn=self.summary_fn, hq_fa=self.pol.icepq.quivered_good_fa, lq_fa=self.pol.icepq.quivered_bad_fa) # Create log file. self.close_log() return 0
class Cluster(IceFiles): """ An object of `Cluster` calls the ICE algorithm to generate consensus isoforms. """ def __init__(self, root_dir, flnc_fa, nfl_fa, bas_fofn, ccs_fofn, out_fa, sge_opts, ice_opts, hq_isoforms_fa=None, hq_isoforms_fq=None, lq_isoforms_fa=None, lq_isoforms_fq=None, report_fn=None, summary_fn=None): super(Cluster, self).__init__(prog_name="Cluster", root_dir=root_dir, bas_fofn=bas_fofn, ccs_fofn=ccs_fofn) self.flnc_fa, self.nfl_fa, self.ccs_fofn = self._validateInputs( flnc_fa, nfl_fa, ccs_fofn) self.root_dir, self.out_fa = self._validateOutputs(root_dir, out_fa) self.hq_isoforms_fa = hq_isoforms_fa self.hq_isoforms_fq = hq_isoforms_fq self.lq_isoforms_fa = lq_isoforms_fa self.lq_isoforms_fq = lq_isoforms_fq self.sge_opts = sge_opts # SGE, CPU options and etc self.ice_opts = ice_opts # The ICE clutering algorithm options self.sanity_check() self._probqv = None # probability & quality value self._flnc_splitted_fas = [] # split flnc_fa into smaller files. self._nflncSplittedFas = [] # split nfl_fa into smaller files. self._logConfigs() # Log configurations self.iceinit = None self.icec = None self.iceq = None self.pol = None self.report_fn = realpath(report_fn) if report_fn is not None \ else op.join(self.root_dir, "cluster_report.csv") self.summary_fn = realpath(summary_fn) if summary_fn is not None \ else op.join(self.root_dir, "cluster_summary.txt") self.summary = ClusterSummary() self.add_log("Finishing create Cluster Object.", level=logging.INFO) def _validateInputs(self, _flnc_fa, _nfl_fa, _ccs_fofn): """Validate input files and return absolute expaneded paths.""" flnc_fa, nfl_fa, ccs_fofn = _flnc_fa, _nfl_fa, _ccs_fofn self.add_log("Checking input files.", level=logging.INFO) if flnc_fa is None or nfl_fa is None: raise ClusterException( "Input non-chimeric reads files needs to be specified.") else: flnc_fa, nfl_fa = realpath(flnc_fa), realpath(nfl_fa) if not op.exists(flnc_fa): raise ClusterException("Unable to find full-length " + "non-chimeric reads: {fn}".format( fn=flnc_fa)) if not op.exists(nfl_fa): raise ClusterException("Unable to find non-full-length " + "non-chimeric reads: {fn}".format( fn=nfl_fa)) if ccs_fofn is not None and not op.exists(ccs_fofn): raise ClusterException("Unable to find FOFN file: " + "{fn}".format(fn=ccs_fofn)) return (flnc_fa, nfl_fa, ccs_fofn) def _validateOutputs(self, _root_dir, _out_fa): """Validate outputs, create root_dir if it does not exist.""" self.add_log("Checking outputs.", level=logging.INFO) root_dir, out_fa = _root_dir, _out_fa if root_dir is None: self.add_log("Output directory needs to be specified.", level=logging.ERROR) if out_fa is None: self.add_log("Output consensus fasta needs to be specified.", level=logging.ERROR) root_dir = realpath(root_dir) out_fa = realpath(out_fa) if op.exists(root_dir): self.add_log( "Output directory {d} already exists.".format(d=root_dir)) else: self.add_log("Creating output directory {d}.".format(d=root_dir)) os.mkdir(root_dir) if op.exists(out_fa): raise ClusterException( "Consensus FASTA file {f} already exists.".format(f=out_fa)) return root_dir, out_fa def sanity_check(self): """Do sanity check before stat to run.""" errMsg = "" if self.ice_opts.quiver is True: if self.bas_fofn is None: errMsg = "A fofn of bas/bax.h5 files, e.g., input.fofn, " + \ "is required in order to polish consensus " + \ "isoforms using quiver." if self.nfl_fa is None: errMsg = "Non-full-length reads are required for polishing " + \ "consensus isoforms using quiver." if errMsg != "": self.add_log(errMsg, level=logging.ERROR) raise ValueError(errMsg) @property def configFN(self): """Return configuration file of the current run.""" return op.join(self.root_dir, "run_ice_config.txt") def _logConfigs(self): """Log configuration.""" with open(self.configFN, 'w', 0) as f: f.write('pbtranscript ' + get_version() + "\n") f.write(str(self.ice_opts) + "\n") f.write(str(self.sge_opts) + "\n") @property def initPickleFN(self): """Return path to pickle file with initial clusters.""" return op.join(self.root_dir, "init.uc.pickle") def _setProbQV(self, ccs_fofn=None, firstSplitFa=None): """Set probability and quality values. If a fofn file is specified, load QV from it. Otherwise, use a pre-defined probability model.""" if ccs_fofn is None: self.add_log("Loading predefined probabilities model.") self._probqv = ProbFromModel(0.01, 0.07, 0.06) else: self.add_log("Loading probabilities and QV from {f}.".format( f=firstSplitFa)) self._probqv = ProbFromQV(ccs_fofn, firstSplitFa) def writeSummary(self, fa, summary_fn, hq_fa=None, lq_fa=None): """Extract number of consensus isoforms predicted, and total number of bases in all consensuus isoforms from fa and write the two attributes to summary_fn. if hq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters if lq_fa (polished high-quality isoforms) is not None, report the number of polished hq clusters """ try: with FastaReader(fa) as reader: for r in reader: self.summary.numConsensusIsoforms += 1 self.summary.numTotalBases += len(r.sequence) if hq_fa is not None: self.summary.num_polished_hq_isoforms = 0 with FastaReader(hq_fa) as reader: for r in reader: self.summary.num_polished_hq_isoforms += 1 if lq_fa is not None: self.summary.num_polished_lq_isoforms = 0 with FastaReader(lq_fa) as reader: for r in reader: self.summary.num_polished_lq_isoforms += 1 self.summary.write(summary_fn) except ZeroDivisionError: errMsg = "No consensus isoforms predicted." self.add_log(errMsg, level=logging.ERROR) raise ClusterException(errMsg) def run(self): """Call ICE to cluster consensus isoforms.""" self.add_log("Start to run cluster.", level=logging.INFO) # Split flnc_fa into smaller files and save files to _flnc_splitted_fas. self.add_log("Splitting {flnc} into ".format(flnc=self.flnc_fa) + "smaller files each containing {n} reads.".format( n=self.ice_opts.flnc_reads_per_split), level=logging.INFO) self._flnc_splitted_fas = splitFasta( input_fasta=self.flnc_fa, reads_per_split=self.ice_opts.flnc_reads_per_split, out_dir=self.root_dir, out_prefix="input.split") self.add_log("Splitted files are: " + "\n".join(self._flnc_splitted_fas), level=logging.INFO) firstSplit = self._flnc_splitted_fas[0] # Set up probabbility and quality value model self._setProbQV(ccs_fofn=self.ccs_fofn, firstSplitFa=firstSplit) # Initialize cluster by clique # check if init.pickle already exists, if so, no need to run IceInit if os.path.exists(self.initPickleFN): self.add_log("{0} already exists. Reading to get uc.".format( self.initPickleFN), level=logging.INFO) with open(self.initPickleFN) as f: uc = cPickle.load(f) else: self.add_log("Finding maximal cliques.", level=logging.INFO) self.iceinit = IceInit(readsFa=firstSplit, qver_get_func=self._probqv.get_smoothed, ice_opts=self.ice_opts, sge_opts=self.sge_opts) uc = self.iceinit.uc # Dump uc to a file self.add_log( "Dumping initial clusters to {f}".format(f=self.initPickleFN), level=logging.INFO) with open(self.initPickleFN, 'w') as f: cPickle.dump(uc, f) # Run IceIterative. self.add_log("Iteratively clustering.", level=logging.INFO) self.icec = IceIterative( fasta_filename=firstSplit, fasta_filenames_to_add=self._flnc_splitted_fas[1:], all_fasta_filename=self.flnc_fa, ccs_fofn=self.ccs_fofn, root_dir=self.root_dir, ice_opts=self.ice_opts, sge_opts=self.sge_opts, uc=uc, probQV=self._probqv) self.icec.run() clean_up_after_ICE(self.root_dir) # IceIterative done, write predicted (unplished) consensus isoforms # to an output fasta self.add_log("Creating a link to unpolished consensus isoforms.") ln(self.icec.final_consensus_fa, self.out_fa) # Call quiver to polish predicted consensus isoforms. if self.ice_opts.quiver is not True: self.add_log("Creating a link to cluster report.") ln(src=self.icec.report_fn, dst=self.report_fn) self.add_log("Writing a summary to {f}".format(f=self.summary_fn), level=logging.INFO) self.writeSummary(fa=self.out_fa, summary_fn=self.summary_fn) else: # self.ice_opts.quiver is True #TODO review code self.pol = Polish(root_dir=self.root_dir, nfl_fa=self.nfl_fa, bas_fofn=self.bas_fofn, ccs_fofn=self.ccs_fofn, hq_isoforms_fa=self.hq_isoforms_fa, hq_isoforms_fq=self.hq_isoforms_fq, lq_isoforms_fa=self.lq_isoforms_fa, lq_isoforms_fq=self.lq_isoforms_fq, ice_opts=self.ice_opts, sge_opts=self.sge_opts) self.pol.run() # cluster report self.add_log("Creating a link to cluster report.") ln(src=self.pol.iceq.report_fn, dst=self.report_fn) # Write a summary. self.add_log("Writing a summary to {f}".format(f=self.summary_fn), level=logging.INFO) self.writeSummary(fa=self.out_fa, summary_fn=self.summary_fn, hq_fa=self.pol.icepq.quivered_good_fa, lq_fa=self.pol.icepq.quivered_bad_fa) # Create log file. self.close_log() return 0