class StreamIEO(object): def __init__(self, logname): """ A class that individually threads the capture of the stdout stream and the stderr stream of a system command. The stdout/stderr are queued in the order they occur. As the que populates another thread, parse the que and prints to the screen using the LogIT class. """ self.io_q = Queue() self.process = None self.streamieolog = LogIt().default(logname="%s - streamieo" % logname, logfile=None) def streamer(self, cmd): self.process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, encoding='utf-8') # Add the command line to the que self.io_q.put(("STDIN", cmd)) # Watch the standard output and add it to the que Thread(target=self._stream_watcher, name='stdout-watcher', args=('STDOUT', self.process.stdout)).start() # Watch the standard error and add it to the que Thread(target=self._stream_watcher, name='stderr-watcher', args=('STDERR', self.process.stderr)).start() # As items are added, print the stream. Thread(target=self._printer, name='_printer').start() def _stream_watcher(self, identifier, stream): # Watch the stream and add to the que dynamically # This runs in tandem with the printer. So as the stdout/stderr streams are queued here, # the que is parsed and printed in the printer function. for line in stream: self.io_q.put((identifier, line)) if not stream.closed: stream.close() def _printer(self): # Prints the que as it is populated with stdout/stderr dynamically. while True: try: # Block for 1 second. item = self.io_q.get(True, 1) except Empty: # No output in either streams for a second. Are we done? if self.process.poll() is not None: break else: identifier, line = item if identifier == "STDIN": self.streamieolog.warn("Command: " + line.strip()) elif identifier == "STDERR": self.streamieolog.error(line.strip()) elif identifier == "STDOUT": self.streamieolog.info(line.strip()) else: self.streamieolog.critical(identifier + ':' + line.strip())
class ClustalO(object): """Align genes using Clustal Omega. This class is a further wrapper around Biopython's ClustalOmegaCommandline. :param infile: Path/Name of multiple fasta file. :param outfile: Path/Name of multiple alignment file. :param logpath: Path to logfile. (Default = None) :param outfmt: Format of the output multiple alignment file. """ def __init__(self, infile, outfile, logpath=None, outfmt="fasta"): """Set up the logger and the parameters.""" self.infile = infile self.outfile = outfile self.outfmt = outfmt self.logpath = logpath self.clustalolog = LogIt().default('clustalo', logfile=self.logpath) def runclustalomega(self): """Run clustalomega.""" try: # Run clustal omega using the multifasta file clustalo_cline = ClustalOmegaCommandline( infile=self.infile, cmd="clustalo", outfile=self.outfile, # "RNA"/"DNA" seqtype="PROTEIN", max_hmm_iterations=2, infmt="fasta", # "aln", "phy" outfmt=self.outfmt, iterations=3, # Notable verbose=True, force=True, log=self.logpath) clustalo_cline() stdout, _ = clustalo_cline() self.clustalolog.info(stdout) except ApplicationError as err: self.clustalolog.error(err)
class BaseSGEJob(object): """Base class for simple jobs.""" def __init__(self, base_jobname, config=None): """Initialize job attributes.""" self.base_jobname = base_jobname if not config: self.default_job_attributes = __DEFAULT__ else: self.default_job_attributes = config self.file2str = file2str self.sgejob_log = LogIt().default(logname="SGE JOB", logfile=None) self.pbsworkdir = os.getcwd() # Import the temp.pbs file using pkg_resources self.temp_pbs = resource_filename(templates.__name__, "temp.pbs") @classmethod def _configure(cls, length, base_jobname): """Configure job attributes or set it up. :param length: :param base_jobname: """ baseid, base = basejobids(length, base_jobname) return baseid, base def debug(self, code): """Debug the SGEJob. :param code: """ raise NotImplementedError def _cleanup(self, jobname): """Clean up job scripts. :param jobname: The name of the job being run or to be run. """ self.sgejob_log.warning('Your job will now be cleaned up.') os.remove(jobname + '.pbs') self.sgejob_log.warning('%s.pbs has been deleted.', jobname) os.remove(jobname + '.py') self.sgejob_log.warning('%s.py has been deleted.' % jobname) def wait_on_job_completion(self, job_id): """Use Qstat to monitor your job. :param job_id: The job id to be monitored. """ # TODO Allow either slack notifications or email or text. qwatch = Qstat().watch(job_id) if qwatch == 'Job id not found.': self.sgejob_log.info('%s has finished.' % job_id) sleep(30) elif qwatch == 'Waiting for %s to start running.' % job_id: self.sgejob_log.info('%s is queued to run.' % job_id) self.sgejob_log.info('Waiting for %s to start.' % job_id) sleep(30) self.wait_on_job_completion(job_id) elif qwatch == 'Waiting for %s to finish running.' % job_id: self.sgejob_log.info('%s is running.' % job_id) self.sgejob_log.info('Waiting for %s to finish.' % job_id) sleep(30) self.wait_on_job_completion(job_id) else: self.wait_on_job_completion(job_id) def submitjob(self, cleanup, wait=True): """Submit a job using qsub. :param cleanup: (Default value = False) :param wait: (Default value = True) """ try: cmd = ['qsub ' + self.jobname + '.pbs'] # this is the command # Shell MUST be True cmd_status = run(cmd, stdout=PIPE, stderr=PIPE, shell=True, check=True) except CalledProcessError as err: self.sgejob_log.error(err.stderr.decode('utf-8')) if cleanup: self._cleanup(self.jobname) else: if cmd_status.returncode == 0: # Command was successful. # The cmd_status has stdout that must be decoded. # When a qsub job is submitted, the stdout is the job id. submitted_jobid = cmd_status.stdout.decode('utf-8') self.sgejob_log.info(self.jobname + ' was submitted.') self.sgejob_log.info('Your job id is: %s' % submitted_jobid) if wait is True: self.wait_on_job_completion(submitted_jobid) self._cleanup(self.jobname) else: # Unsuccessful. Stdout will be '1' self.sgejob_log.error('PBS job not submitted.')