Example #1
0
 def endFork(self,fdict):   ### Ends fork, tidies and sets new one running
     '''Ends fork, tidies and sets new one running.'''
     try:### ~ [1] ~ End and tidy current job ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ###
         if 'ResFile' in fdict:
             for resfile in fdict['ResFile']:
                 fromfile = '%s.%s' % (fdict['FID'],resfile)
                 if not rje.exists(fromfile): continue #self.warnLog('Results file %s missing!' % fromfile); continue
                 tofile = '%s.%s' % (self.baseFile(),resfile)
                 if rje.exists(tofile): open(tofile,'a').writelines(open(fromfile,'r').readlines()[1:])
                 else: rje.fileTransfer(fromfile,tofile)
         if 'Log' in fdict:
             if 'cmd' in fdict:
                 open(self.log.info['LogFile'],'a').writelines(open(fdict['Log'],'r').readlines()[5:-1])
                 os.unlink(fdict['Log'])
             else: rje.fileTransfer(fdict['Log'],self.log.info['LogFile'])
             if self.getBool('LogFork'):
                 self.printLog('#END','Fork %s ended: log content transferred' % fdict['PID'])
                 self.printLog('#~~#','#~~#',timeout=False)
             #if self.dev(): self.deBug(fdict['Log'])
             #if self.dev(): self.deBug(rje.exists(fdict['Log']))
         elif 'PID' in fdict and string.split('%s' % fdict['PID'])[0] == 'WAIT': pass
         else: self.printLog('#END','Fork %s ended.' % fdict['PID'])
     except IOError:
         if self.getInt('IOError') == 1: self.errorLog('Forker.endFork IOError limit reached'); raise
         else: self.int['IOError'] -= 1; self.errorLog('Forker.endFork')
     except: self.errorLog('Forker.endFork error')
     self.nextFork()   # Carry on regardless
Example #2
0
 def endFork(self, fdict):  ### Ends fork, tidies and sets new one running
     '''Ends fork, tidies and sets new one running.'''
     try:  ### ~ [1] ~ End and tidy current job ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ###
         if 'ResFile' in fdict:
             for resfile in fdict['ResFile']:
                 fromfile = '%s.%s' % (fdict['FID'], resfile)
                 if not rje.exists(fromfile):
                     continue  #self.warnLog('Results file %s missing!' % fromfile); continue
                 tofile = '%s.%s' % (self.baseFile(), resfile)
                 if rje.exists(tofile):
                     open(tofile, 'a').writelines(
                         open(fromfile, 'r').readlines()[1:])
                     os.unlink(fromfile)
                 else:
                     rje.fileTransfer(fromfile, tofile)
         if 'Log' in fdict:
             if 'cmd' in fdict:
                 open(self.log.info['LogFile'], 'a').writelines(
                     open(fdict['Log'], 'r').readlines()[5:-1])
                 os.unlink(fdict['Log'])
             else:
                 rje.fileTransfer(fdict['Log'], self.log.info['LogFile'])
             if self.getBool('LogFork'):
                 self.printLog(
                     '#END', 'Fork %s ended: log content transferred' %
                     fdict['PID'])
                 self.printLog('#~~#', '#~~#', timeout=False)
             #if self.dev(): self.deBug(fdict['Log'])
             #if self.dev(): self.deBug(rje.exists(fdict['Log']))
         elif 'PID' in fdict and string.split(
                 '%s' % fdict['PID'])[0] == 'WAIT':
             pass
         else:
             self.printLog('#END',
                           'Fork %s ended.' % fdict['PID'],
                           log=self.getBool('LogFork'),
                           screen=self.getBool('LogFork') or self.v() > 1)
     except IOError:
         if self.getInt('IOError') == 1:
             self.errorLog('Forker.endFork IOError limit reached')
             raise
         else:
             self.int['IOError'] -= 1
             self.errorLog('Forker.endFork')
     except:
         self.errorLog('Forker.endFork error')
     self.nextFork()  # Carry on regardless
Example #3
0
    def forker(self):      ### Generic forking method
        '''
        Generic method for forking (without threads).
        Add description here (and arguments.)
        '''
        try:
            ### <0> ### Setup
            _stage = '<0> Fork Setup'
            forkx = int(self.stat['Forks'])   # Number of forks to have running at one time
            if self.opt['Win32'] or forkx < 1:
                self.opt['NoForks'] = True
            forks = []  # List of active fork PIDs
            killforks = int(self.stat['KillForks']) # Time in seconds to wait after main thread has apparently finished
            forking_condition = True    # Condition to keep forking

            ### Sequence List setup ###
            _stage = '<1> Forking'
            seqx = 0    # Sequence Counter
            subx = 0    # Subset sequence counter
            outfile = None  # Output file name
            randlist = []   # List of random strings for split sequence files
            filedict = {}   # Dictionary of input files for each random string
            seqlist = rje_seq.SeqList(log=self.log,cmd_list=['autoload=F']+self.cmd_list)
            seqlist.makeBaseFile()
            SEQFILE = open(seqlist.info['Name'], 'r')
            (seq,lastline) = seqlist.nextFasSeq(SEQFILE,'Starting')
            while seq:
                seqlist.seq = [seq]
                if self.info['StartFrom'] != 'None':    # Not yet reached wanted sequence
                    if self.info['StartFrom'] in [seq.info['Name'], seq.info['ID'], seq.info['AccNum'], seq.shortName()]:
                        self.info['StartFrom'] = 'None'
                if self.info['StartFrom'] == 'None':    # Wanted sequence
                    if outfile:   # Create new file
                        SEQOUT = open(outfile,'a')
                    else:
                        rs = rje.randomString(6)
                        while rs in randlist:
                            rs = rje.randomString(6)
                        outfile = '%s.%s.fas' % (seqlist.info['Basefile'],rs)
                        SEQOUT = open(outfile,'w')
                        randlist.append(rs)
                        filedict[rs] = outfile
                    SEQOUT.write('>%s\n%s\n' % (seq.info['Name'],seq.info['Sequence']))
                    SEQOUT.close()
                    seqx += 1
                    subx += 1
                    if subx == self.stat['Split']:  # Finished split
                        self.log.printLog('#SEQ','%s sequences output to %s.' % (rje.integerString(subx),outfile))
                        outfile = None
                        subx = 0
                (seq,lastline) = seqlist.nextFasSeq(SEQFILE,lastline)
            if subx > 0:
                self.log.printLog('#SEQ','%s sequences output to %s.' % (rje.integerString(subx),outfile))
            self.log.printLog('#SEQ','%s sequences output in total to %d files.' % (rje.integerString(seqx),len(randlist)))
            # Now have the list of random strings in randlist (in order) and filenames in filedict

            ### <1> ### Forking                            
            killtime = time.time()
            dealt_with = 0      # Split files dealt with
            while dealt_with < len(randlist) or len(forks):

                ## <a> ## forks
                _stage = '<1a> New Forks'
                while dealt_with < len(randlist) and (len(forks) < forkx or self.opt['NoForks']):     # Add more forks
                    _stage = '<1a-i> Fork: Get stuff for fork'
                    killtime = time.time()  # Reset killtime - still doing stuff
                    # Add new fork
                    _stage = '<1a-ii> Fork: New Fork'
                    new_fork_id = randlist[dealt_with]
                    dealt_with += 1
                    
                    outcmd = string.split(self.info['OutCmd'],'.')
                    if len(outcmd) > 1:
                        outcmd = outcmd[:-1] + [new_fork_id] + outcmd[-1:]
                    else:
                        outcmd = outcmd + [new_fork_id] + ['resfile']
                    outcmd = string.join(outcmd,'.')    
                    forkcmd = '%s %s%s %s %s log=%s.log newlog=T i=-1' % (self.info['ForkProg'],self.info['SeqInCmd'],filedict[new_fork_id],outcmd,self.info['ForkCmd'],new_fork_id)
                    if self.opt['NoForks']:
                        os.system(forkcmd)                        
                    else:   # Forks
                        newpid = os.fork() 
                        if newpid == 0: # child 
                            os.system(forkcmd)
                            sys.exit()    # Exit process 
                        elif newpid == -1: # error
                            self.log.errorLog('Problem forking %s.' % new_fork_id)
                        else: 
                            forks.append(newpid)    # Add fork to list 
            
                ## <b> ## Monitor and remove finished forks
                _stage = '<1b> Finished Forks'
                forklist = self._activeForks(forks)
                if len(forklist) != len(forks):
                    self.verbose(0,2,' => %d of %d forks finished!' % (len(forks) - len(forklist),len(forks)),1)
                    forks = forklist[0:]
                        
                self.verbose(3,3,'End of a Cycle.',2)

                ## <c> ## Look for eternal hanging of forks
                _stage = '<1c> Hanging'
                if time.time() - killtime > killforks:
                    self.verbose(0,1,'\n%d seconds of main program inactivity. %d forks still active!' % (killforks,len(forks)),1)
                    for fork in forks:
                        self.verbose(0,2,' => Fork PID %d still Active!' % (fork),1)
                    if rje.yesNo('Kill?'):
                        break   #!# killing options
                    else:
                        killtime = time.time()

            ### <3> ### Finish
            _stage = '<3> Finish'
            if len(forks) > 0:
                self.log.errorLog('%d Forks still active after %d seconds of main program inactivity' % (len(forks),killforks),True)
            else:
                self.verbose(0,1,'Forks have finished.',2)

            ### <4> ### Recompile results
            for randstr in randlist:
                os.unlink(filedict[randstr])
                rje.fileTransfer(fromfile='%s.log' % randstr,tofile=self.log.info['Name'],deletefrom=True)            
                outfiles = glob.glob('*.%s.*' % randstr)
                for outfile in outfiles:
                    compfile = outfile.split('.')
                    compfile.remove(randstr)
                    compfile = string.join(compfile,'.')
                    if randstr == randlist[0] and os.path.exists(compfile) and not self.opt['Append']:
                        os.unlink(compfile)
                    rje.fileTransfer(fromfile=outfile,tofile=compfile,deletefrom=True)
                    self.verbose(1,2,'Copying results data from %s to %s...' % (outfile,compfile),0)
                self.verbose(0,1,'%d results files copied for Split %d.' % (len(outfiles),(randlist.index(randstr)+1)),1)
            self.log.printLog('#OUT','Results for %d splits compiled.' % len(randlist))
            
        except SystemExit:  # Don't want forks raising an Exception upon exiting
            sys.exit()
        except:
            self.log.errorLog('Error in forker(%s):' % _stage,printerror=True,quitchoice=False)     
            raise   # Delete this if method error not terrible