def endFork(self,fdict): ### Ends fork, tidies and sets new one running
'''Ends fork, tidies and sets new one running.'''
try:### ~ [1] ~ End and tidy current job ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ###
if 'ResFile' in fdict:
for resfile in fdict['ResFile']:
fromfile = '%s.%s' % (fdict['FID'],resfile)
if not rje.exists(fromfile): continue #self.warnLog('Results file %s missing!' % fromfile); continue
tofile = '%s.%s' % (self.baseFile(),resfile)
if rje.exists(tofile): open(tofile,'a').writelines(open(fromfile,'r').readlines()[1:])
else: rje.fileTransfer(fromfile,tofile)
if 'Log' in fdict:
if 'cmd' in fdict:
open(self.log.info['LogFile'],'a').writelines(open(fdict['Log'],'r').readlines()[5:-1])
os.unlink(fdict['Log'])
else: rje.fileTransfer(fdict['Log'],self.log.info['LogFile'])
if self.getBool('LogFork'):
self.printLog('#END','Fork %s ended: log content transferred' % fdict['PID'])
self.printLog('#~~#','#~~#',timeout=False)
#if self.dev(): self.deBug(fdict['Log'])
#if self.dev(): self.deBug(rje.exists(fdict['Log']))
elif 'PID' in fdict and string.split('%s' % fdict['PID'])[0] == 'WAIT': pass
else: self.printLog('#END','Fork %s ended.' % fdict['PID'])
except IOError:
if self.getInt('IOError') == 1: self.errorLog('Forker.endFork IOError limit reached'); raise
else: self.int['IOError'] -= 1; self.errorLog('Forker.endFork')
except: self.errorLog('Forker.endFork error')
self.nextFork() # Carry on regardless
def endFork(self, fdict): ### Ends fork, tidies and sets new one running
'''Ends fork, tidies and sets new one running.'''
try: ### ~ [1] ~ End and tidy current job ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ###
if 'ResFile' in fdict:
for resfile in fdict['ResFile']:
fromfile = '%s.%s' % (fdict['FID'], resfile)
if not rje.exists(fromfile):
continue #self.warnLog('Results file %s missing!' % fromfile); continue
tofile = '%s.%s' % (self.baseFile(), resfile)
if rje.exists(tofile):
open(tofile, 'a').writelines(
open(fromfile, 'r').readlines()[1:])
os.unlink(fromfile)
else:
rje.fileTransfer(fromfile, tofile)
if 'Log' in fdict:
if 'cmd' in fdict:
open(self.log.info['LogFile'], 'a').writelines(
open(fdict['Log'], 'r').readlines()[5:-1])
os.unlink(fdict['Log'])
else:
rje.fileTransfer(fdict['Log'], self.log.info['LogFile'])
if self.getBool('LogFork'):
self.printLog(
'#END', 'Fork %s ended: log content transferred' %
fdict['PID'])
self.printLog('#~~#', '#~~#', timeout=False)
#if self.dev(): self.deBug(fdict['Log'])
#if self.dev(): self.deBug(rje.exists(fdict['Log']))
elif 'PID' in fdict and string.split(
'%s' % fdict['PID'])[0] == 'WAIT':
pass
else:
self.printLog('#END',
'Fork %s ended.' % fdict['PID'],
log=self.getBool('LogFork'),
screen=self.getBool('LogFork') or self.v() > 1)
except IOError:
if self.getInt('IOError') == 1:
self.errorLog('Forker.endFork IOError limit reached')
raise
else:
self.int['IOError'] -= 1
self.errorLog('Forker.endFork')
except:
self.errorLog('Forker.endFork error')
self.nextFork() # Carry on regardless
def forker(self): ### Generic forking method
'''
Generic method for forking (without threads).
Add description here (and arguments.)
'''
try:
### <0> ### Setup
_stage = '<0> Fork Setup'
forkx = int(self.stat['Forks']) # Number of forks to have running at one time
if self.opt['Win32'] or forkx < 1:
self.opt['NoForks'] = True
forks = [] # List of active fork PIDs
killforks = int(self.stat['KillForks']) # Time in seconds to wait after main thread has apparently finished
forking_condition = True # Condition to keep forking
### Sequence List setup ###
_stage = '<1> Forking'
seqx = 0 # Sequence Counter
subx = 0 # Subset sequence counter
outfile = None # Output file name
randlist = [] # List of random strings for split sequence files
filedict = {} # Dictionary of input files for each random string
seqlist = rje_seq.SeqList(log=self.log,cmd_list=['autoload=F']+self.cmd_list)
seqlist.makeBaseFile()
SEQFILE = open(seqlist.info['Name'], 'r')
(seq,lastline) = seqlist.nextFasSeq(SEQFILE,'Starting')
while seq:
seqlist.seq = [seq]
if self.info['StartFrom'] != 'None': # Not yet reached wanted sequence
if self.info['StartFrom'] in [seq.info['Name'], seq.info['ID'], seq.info['AccNum'], seq.shortName()]:
self.info['StartFrom'] = 'None'
if self.info['StartFrom'] == 'None': # Wanted sequence
if outfile: # Create new file
SEQOUT = open(outfile,'a')
else:
rs = rje.randomString(6)
while rs in randlist:
rs = rje.randomString(6)
outfile = '%s.%s.fas' % (seqlist.info['Basefile'],rs)
SEQOUT = open(outfile,'w')
randlist.append(rs)
filedict[rs] = outfile
SEQOUT.write('>%s\n%s\n' % (seq.info['Name'],seq.info['Sequence']))
SEQOUT.close()
seqx += 1
subx += 1
if subx == self.stat['Split']: # Finished split
self.log.printLog('#SEQ','%s sequences output to %s.' % (rje.integerString(subx),outfile))
outfile = None
subx = 0
(seq,lastline) = seqlist.nextFasSeq(SEQFILE,lastline)
if subx > 0:
self.log.printLog('#SEQ','%s sequences output to %s.' % (rje.integerString(subx),outfile))
self.log.printLog('#SEQ','%s sequences output in total to %d files.' % (rje.integerString(seqx),len(randlist)))
# Now have the list of random strings in randlist (in order) and filenames in filedict
### <1> ### Forking
killtime = time.time()
dealt_with = 0 # Split files dealt with
while dealt_with < len(randlist) or len(forks):
## <a> ## forks
_stage = '<1a> New Forks'
while dealt_with < len(randlist) and (len(forks) < forkx or self.opt['NoForks']): # Add more forks
_stage = '<1a-i> Fork: Get stuff for fork'
killtime = time.time() # Reset killtime - still doing stuff
# Add new fork
_stage = '<1a-ii> Fork: New Fork'
new_fork_id = randlist[dealt_with]
dealt_with += 1
outcmd = string.split(self.info['OutCmd'],'.')
if len(outcmd) > 1:
outcmd = outcmd[:-1] + [new_fork_id] + outcmd[-1:]
else:
outcmd = outcmd + [new_fork_id] + ['resfile']
outcmd = string.join(outcmd,'.')
forkcmd = '%s %s%s %s %s log=%s.log newlog=T i=-1' % (self.info['ForkProg'],self.info['SeqInCmd'],filedict[new_fork_id],outcmd,self.info['ForkCmd'],new_fork_id)
if self.opt['NoForks']:
os.system(forkcmd)
else: # Forks
newpid = os.fork()
if newpid == 0: # child
os.system(forkcmd)
sys.exit() # Exit process
elif newpid == -1: # error
self.log.errorLog('Problem forking %s.' % new_fork_id)
else:
forks.append(newpid) # Add fork to list
## <b> ## Monitor and remove finished forks
_stage = '<1b> Finished Forks'
forklist = self._activeForks(forks)
if len(forklist) != len(forks):
self.verbose(0,2,' => %d of %d forks finished!' % (len(forks) - len(forklist),len(forks)),1)
forks = forklist[0:]
self.verbose(3,3,'End of a Cycle.',2)
## <c> ## Look for eternal hanging of forks
_stage = '<1c> Hanging'
if time.time() - killtime > killforks:
self.verbose(0,1,'\n%d seconds of main program inactivity. %d forks still active!' % (killforks,len(forks)),1)
for fork in forks:
self.verbose(0,2,' => Fork PID %d still Active!' % (fork),1)
if rje.yesNo('Kill?'):
break #!# killing options
else:
killtime = time.time()
### <3> ### Finish
_stage = '<3> Finish'
if len(forks) > 0:
self.log.errorLog('%d Forks still active after %d seconds of main program inactivity' % (len(forks),killforks),True)
else:
self.verbose(0,1,'Forks have finished.',2)
### <4> ### Recompile results
for randstr in randlist:
os.unlink(filedict[randstr])
rje.fileTransfer(fromfile='%s.log' % randstr,tofile=self.log.info['Name'],deletefrom=True)
outfiles = glob.glob('*.%s.*' % randstr)
for outfile in outfiles:
compfile = outfile.split('.')
compfile.remove(randstr)
compfile = string.join(compfile,'.')
if randstr == randlist[0] and os.path.exists(compfile) and not self.opt['Append']:
os.unlink(compfile)
rje.fileTransfer(fromfile=outfile,tofile=compfile,deletefrom=True)
self.verbose(1,2,'Copying results data from %s to %s...' % (outfile,compfile),0)
self.verbose(0,1,'%d results files copied for Split %d.' % (len(outfiles),(randlist.index(randstr)+1)),1)
self.log.printLog('#OUT','Results for %d splits compiled.' % len(randlist))
except SystemExit: # Don't want forks raising an Exception upon exiting
sys.exit()
except:
self.log.errorLog('Error in forker(%s):' % _stage,printerror=True,quitchoice=False)
raise # Delete this if method error not terrible