class StreamIEO(object):
def __init__(self, logname):
"""
A class that individually threads the capture of the stdout stream and the stderr stream of a system command.
The stdout/stderr are queued in the order they occur. As the que populates another thread, parse the que and
prints to the screen using the LogIT class.
"""
self.io_q = Queue()
self.process = None
self.streamieolog = LogIt().default(logname="%s - streamieo" % logname,
logfile=None)
def streamer(self, cmd):
self.process = Popen(cmd,
stdout=PIPE,
stderr=PIPE,
shell=True,
encoding='utf-8')
# Add the command line to the que
self.io_q.put(("STDIN", cmd))
# Watch the standard output and add it to the que
Thread(target=self._stream_watcher,
name='stdout-watcher',
args=('STDOUT', self.process.stdout)).start()
# Watch the standard error and add it to the que
Thread(target=self._stream_watcher,
name='stderr-watcher',
args=('STDERR', self.process.stderr)).start()
# As items are added, print the stream.
Thread(target=self._printer, name='_printer').start()
def _stream_watcher(self, identifier, stream):
# Watch the stream and add to the que dynamically
# This runs in tandem with the printer. So as the stdout/stderr streams are queued here,
# the que is parsed and printed in the printer function.
for line in stream:
self.io_q.put((identifier, line))
if not stream.closed:
stream.close()
def _printer(self):
# Prints the que as it is populated with stdout/stderr dynamically.
while True:
try:
# Block for 1 second.
item = self.io_q.get(True, 1)
except Empty:
# No output in either streams for a second. Are we done?
if self.process.poll() is not None:
break
else:
identifier, line = item
if identifier == "STDIN":
self.streamieolog.warn("Command: " + line.strip())
elif identifier == "STDERR":
self.streamieolog.error(line.strip())
elif identifier == "STDOUT":
self.streamieolog.info(line.strip())
else:
self.streamieolog.critical(identifier + ':' + line.strip())
class ClustalO(object):
"""Align genes using Clustal Omega.
This class is a further wrapper around Biopython's ClustalOmegaCommandline.
:param infile: Path/Name of multiple fasta file.
:param outfile: Path/Name of multiple alignment file.
:param logpath: Path to logfile. (Default = None)
:param outfmt: Format of the output multiple alignment file.
"""
def __init__(self, infile, outfile, logpath=None, outfmt="fasta"):
"""Set up the logger and the parameters."""
self.infile = infile
self.outfile = outfile
self.outfmt = outfmt
self.logpath = logpath
self.clustalolog = LogIt().default('clustalo', logfile=self.logpath)
def runclustalomega(self):
"""Run clustalomega."""
try:
# Run clustal omega using the multifasta file
clustalo_cline = ClustalOmegaCommandline(
infile=self.infile,
cmd="clustalo",
outfile=self.outfile,
# "RNA"/"DNA"
seqtype="PROTEIN",
max_hmm_iterations=2,
infmt="fasta",
# "aln", "phy"
outfmt=self.outfmt,
iterations=3, # Notable
verbose=True,
force=True,
log=self.logpath)
clustalo_cline()
stdout, _ = clustalo_cline()
self.clustalolog.info(stdout)
except ApplicationError as err:
self.clustalolog.error(err)
class BaseSGEJob(object):
"""Base class for simple jobs."""
def __init__(self, base_jobname, config=None):
"""Initialize job attributes."""
self.base_jobname = base_jobname
if not config:
self.default_job_attributes = __DEFAULT__
else:
self.default_job_attributes = config
self.file2str = file2str
self.sgejob_log = LogIt().default(logname="SGE JOB", logfile=None)
self.pbsworkdir = os.getcwd()
# Import the temp.pbs file using pkg_resources
self.temp_pbs = resource_filename(templates.__name__, "temp.pbs")
@classmethod
def _configure(cls, length, base_jobname):
"""Configure job attributes or set it up.
:param length:
:param base_jobname:
"""
baseid, base = basejobids(length, base_jobname)
return baseid, base
def debug(self, code):
"""Debug the SGEJob.
:param code:
"""
raise NotImplementedError
def _cleanup(self, jobname):
"""Clean up job scripts.
:param jobname: The name of the job being run or to be run.
"""
self.sgejob_log.warning('Your job will now be cleaned up.')
os.remove(jobname + '.pbs')
self.sgejob_log.warning('%s.pbs has been deleted.', jobname)
os.remove(jobname + '.py')
self.sgejob_log.warning('%s.py has been deleted.' % jobname)
def wait_on_job_completion(self, job_id):
"""Use Qstat to monitor your job.
:param job_id: The job id to be monitored.
"""
# TODO Allow either slack notifications or email or text.
qwatch = Qstat().watch(job_id)
if qwatch == 'Job id not found.':
self.sgejob_log.info('%s has finished.' % job_id)
sleep(30)
elif qwatch == 'Waiting for %s to start running.' % job_id:
self.sgejob_log.info('%s is queued to run.' % job_id)
self.sgejob_log.info('Waiting for %s to start.' % job_id)
sleep(30)
self.wait_on_job_completion(job_id)
elif qwatch == 'Waiting for %s to finish running.' % job_id:
self.sgejob_log.info('%s is running.' % job_id)
self.sgejob_log.info('Waiting for %s to finish.' % job_id)
sleep(30)
self.wait_on_job_completion(job_id)
else:
self.wait_on_job_completion(job_id)
def submitjob(self, cleanup, wait=True):
"""Submit a job using qsub.
:param cleanup: (Default value = False)
:param wait: (Default value = True)
"""
try:
cmd = ['qsub ' + self.jobname + '.pbs'] # this is the command
# Shell MUST be True
cmd_status = run(cmd, stdout=PIPE, stderr=PIPE, shell=True, check=True)
except CalledProcessError as err:
self.sgejob_log.error(err.stderr.decode('utf-8'))
if cleanup:
self._cleanup(self.jobname)
else:
if cmd_status.returncode == 0: # Command was successful.
# The cmd_status has stdout that must be decoded.
# When a qsub job is submitted, the stdout is the job id.
submitted_jobid = cmd_status.stdout.decode('utf-8')
self.sgejob_log.info(self.jobname + ' was submitted.')
self.sgejob_log.info('Your job id is: %s' % submitted_jobid)
if wait is True:
self.wait_on_job_completion(submitted_jobid)
self._cleanup(self.jobname)
else: # Unsuccessful. Stdout will be '1'
self.sgejob_log.error('PBS job not submitted.')
