예제 #1
0
파일: refine.py 프로젝트: VuisterLab/cing
def analyze(config, project, parameters, doPrint=0):
    '''Analyze a run
    Returns True on failure.
    '''
    nTmessage("\n-- analyze --")

    inPath = config.directories.converted
    modelList = asci2list(parameters.models)
    if getDeepByKeysOrAttributes(parameters, USE_ANNEALED_STR):
        inPath = config.directories.annealed
        modelList = asci2list(parameters.modelsAnneal)

    nTdebug('inPath:         %s' % inPath)
    nTdebug('modelList:      %s' % modelList)

    # first create the jobs, run later
    analyzeJobs = []
    for i in modelList:
        job = Analyze(
            config,
            parameters,
            project=project,
            fileNum=i,
            molecules=[
                NTdict(psfFile=parameters.psfFile,
                       pdbFile=parameters.baseName % i,
                       selectionCode='(not resn TIP3 and not resn ANI)'),
            ],
            inPath=inPath,
            outPath=config.directories.analyzed,
            jobName='analyze_%d' % i,
        )

        analyzeJobs.append(job)
    #end for

    job_list = []
    for job in analyzeJobs:
        if job.createScript():
            nTerror(
                "In refine#analyze failed to create at least one job's script."
            )
            return True
        if doPrint:
            job.printScript()
        #end if
#        job.runScript()
        job_list.append((job.runScript, ))
    #end for
    f = ForkOff(processes_max=cing.ncpus,
                max_time_to_wait=600,
                verbosity=cing.verbosity)
    done_list = f.forkoff_start(job_list, 0)  # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    if not done_list:
        notDone = parameters.modelCountAnneal - len(done_list)
        nTerror("Failed to analyze %s", notDone)
        return True
예제 #2
0
파일: refine.py 프로젝트: jakesyl/cing
def anneal(config, project, parameters, doPrint=0):
    nTmessage("\n-- anneal --")
    # first create the jobs, run later
    annealJobs = []
    for i in range(parameters.modelCountAnneal):
        annealJobs.append(
            Anneal(
                   config,
                   parameters,
                   project = project,
                   fileNum=i,
                   molecules=[
                                 NTdict(
                                    psfFile=parameters.psfFile,
                                    pdbFile=parameters.baseName % i,
                                    selectionCode='(all)'
                                 ),
                                ],
                   inPath     = config.directories.template,
                   outPath    = config.directories.annealed,
                   templateFile = TEMPLATE_FILE_NAME,
                   jobName='anneal_%d' % i,
            )
        )

    job_list = []
    for job in annealJobs:
        job.createScript()
        if doPrint:
            job.printScript()
        job_list.append((job.runScript,) )
#        job.runScript()
    #end for
    f = ForkOff(
            processes_max=cing.ncpus,
            max_time_to_wait=600,
            verbosity=cing.verbosity
        )
    done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    notDone = parameters.modelCountAnneal - len(done_list)
    if notDone > 0:
        nTwarning("Failed to anneal %s", notDone) # We print how many models failed to anneal, and try to continue without them
        nTdebug("Only finished ids will be kept")
    parameters.modelsAnneal = list2asci(done_list)

    # Only if the list is empty, we want to return True
    if not done_list:
        return True
예제 #3
0
파일: refine.py 프로젝트: VuisterLab/cing
def anneal(config, project, parameters, doPrint=0):
    nTmessage("\n-- anneal --")
    # first create the jobs, run later
    annealJobs = []
    for i in range(parameters.modelCountAnneal):
        annealJobs.append(
            Anneal(
                config,
                parameters,
                project=project,
                fileNum=i,
                molecules=[
                    NTdict(psfFile=parameters.psfFile,
                           pdbFile=parameters.baseName % i,
                           selectionCode='(all)'),
                ],
                inPath=config.directories.template,
                outPath=config.directories.annealed,
                templateFile=TEMPLATE_FILE_NAME,
                jobName='anneal_%d' % i,
            ))

    job_list = []
    for job in annealJobs:
        job.createScript()
        if doPrint:
            job.printScript()
        job_list.append((job.runScript, ))
#        job.runScript()
#end for
    f = ForkOff(processes_max=cing.ncpus,
                max_time_to_wait=600,
                verbosity=cing.verbosity)
    done_list = f.forkoff_start(job_list, 0)  # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    notDone = parameters.modelCountAnneal - len(done_list)
    if notDone > 0:
        nTwarning(
            "Failed to anneal %s", notDone
        )  # We print how many models failed to anneal, and try to continue without them
        nTdebug("Only finished ids will be kept")
    parameters.modelsAnneal = list2asci(done_list)

    # Only if the list is empty, we want to return True
    if not done_list:
        return True
예제 #4
0
파일: main.py 프로젝트: VuisterLab/cing
def testOverall(namepattern):
    'Use silent testing from top level.'
#    cing.verbosity = verbosityError
    # Add the ones you don't want to test (perhaps you know they don't work yet)
    excludedModuleList = [ 
#                           cingPythonDir + "/Cython*",
                           cingPythonDir + "/cyana2cing*",
#                           cingPythonDir + "/cing.PluginCode",
#                           cingPythonDir + "/cing.PluginCode.test.test_Whatif",
#                           cingPythonDir + "/cing.Scripts.test.test_cyana2cing",
#                           cingPythonDir + "/cing.STAR.FileTest",
                          ]
    startdir = cingPythonDir
    nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
    # enable next line(s) to do a single check only.
#    nameList = ['/Users/jd/workspace35/cing/python/cing/Libs/test/test_Imagery.py']
#    nameList = [
#                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
#                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py'
#                ]
    nTdebug('Will unit check: ' + repr(nameList))
#    nameList = nameList[0:5]
#    namepattern = "*Test.py"
#    nameList2 = findFiles(namepattern, startdir)
#    for name in nameList2:
#      nameList.append(name)
    # translate: '/Users/jd/workspace/cing/python/cing/Libs/test/test_NTplot.py'
    # to: cing.Libs.test.test_NTplot

    f = ForkOff(
#            processes_max=1,     # use 1 if you really want to read it line by line.
            processes_max=cing.ncpus,
            max_time_to_wait=600, # on a very slow setup
            verbosity=cing.verbosity
            )
    job_list = []
    for name in nameList:
        job_list.append( (testByName, (name, excludedModuleList)) )
    done_list = f.forkoff_start(job_list, 0)
    nTmessage("Finished ids: %s", done_list)

    # Exit with timer info anywho. After this CING should exit so the tweak shouldn't break anything.
    if cing.verbosity <= cing.verbosityError:
        cing.verbosity = cing.verbosityOutput
예제 #5
0
def testOverall(namepattern):
    'Use silent testing from top level.'
    #    cing.verbosity = verbosityError
    # Add the ones you don't want to test (perhaps you know they don't work yet)
    excludedModuleList = [
        #                           cingPythonDir + "/Cython*",
        cingPythonDir + "/cyana2cing*",
        #                           cingPythonDir + "/cing.PluginCode",
        #                           cingPythonDir + "/cing.PluginCode.test.test_Whatif",
        #                           cingPythonDir + "/cing.Scripts.test.test_cyana2cing",
        #                           cingPythonDir + "/cing.STAR.FileTest",
    ]
    startdir = cingPythonDir
    nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
    # enable next line(s) to do a single check only.
    #    nameList = ['/Users/jd/workspace35/cing/python/cing/Libs/test/test_Imagery.py']
    #    nameList = [
    #                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
    #                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py'
    #                ]
    nTdebug('Will unit check: ' + repr(nameList))
    #    nameList = nameList[0:5]
    #    namepattern = "*Test.py"
    #    nameList2 = findFiles(namepattern, startdir)
    #    for name in nameList2:
    #      nameList.append(name)
    # translate: '/Users/jd/workspace/cing/python/cing/Libs/test/test_NTplot.py'
    # to: cing.Libs.test.test_NTplot

    f = ForkOff(
        #            processes_max=1,     # use 1 if you really want to read it line by line.
        processes_max=cing.ncpus,
        max_time_to_wait=600,  # on a very slow setup
        verbosity=cing.verbosity)
    job_list = []
    for name in nameList:
        job_list.append((testByName, (name, excludedModuleList)))
    done_list = f.forkoff_start(job_list, 0)
    nTmessage("Finished ids: %s", done_list)

    # Exit with timer info anywho. After this CING should exit so the tweak shouldn't break anything.
    if cing.verbosity <= cing.verbosityError:
        cing.verbosity = cing.verbosityOutput
예제 #6
0
파일: refine.py 프로젝트: jakesyl/cing
def refine(config, project, parameters, doPrint=0):
    nTmessage("\n-- refine --")
    # first create the jobs, run later
    refineJobs = []
    for i in asci2list(parameters.models):
        refineJobs.append(
            WaterRefine(
                   config,
                   parameters,
                   project = project,
                   fileNum=i,
                   molecules=[
                                 NTdict(
                                    psfFile=parameters.psfFile,
                                    pdbFile=parameters.baseName % i,
                                    selectionCode='(not resn TIP3 and not resn ANI)'
                                 ),
                                ],
                   inPath     = config.directories.analyzed,
                   outPath    = config.directories.refined,
                   jobName='refine_%d' % i,
            )
        )

    job_list = []
    for job in refineJobs:
        job.createScript()
        if doPrint:
            job.printScript()
#        job.runScript()
        job_list.append((job.runScript,) )
    #end for
    f = ForkOff(
            processes_max=cing.ncpus,
            max_time_to_wait=6000,
            verbosity=cing.verbosity
        )
    done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    if not done_list:
        notDone = parameters.modelCountAnneal - len(done_list)
        nTerror("Failed to refine %s", notDone)
        return True
예제 #7
0
파일: refine.py 프로젝트: VuisterLab/cing
def refine(config, project, parameters, doPrint=0):
    nTmessage("\n-- refine --")
    # first create the jobs, run later
    refineJobs = []
    for i in asci2list(parameters.models):
        refineJobs.append(
            WaterRefine(
                config,
                parameters,
                project=project,
                fileNum=i,
                molecules=[
                    NTdict(psfFile=parameters.psfFile,
                           pdbFile=parameters.baseName % i,
                           selectionCode='(not resn TIP3 and not resn ANI)'),
                ],
                inPath=config.directories.analyzed,
                outPath=config.directories.refined,
                jobName='refine_%d' % i,
            ))

    job_list = []
    for job in refineJobs:
        job.createScript()
        if doPrint:
            job.printScript()
#        job.runScript()
        job_list.append((job.runScript, ))
    #end for
    f = ForkOff(processes_max=cing.ncpus,
                max_time_to_wait=6000,
                verbosity=cing.verbosity)
    done_list = f.forkoff_start(job_list, 0)  # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    if not done_list:
        notDone = parameters.modelCountAnneal - len(done_list)
        nTerror("Failed to refine %s", notDone)
        return True
예제 #8
0
def doPylintOverall(pylintFileName='pylint.txt'):
    "Add the ones you don't want to pylint (perhaps you know they don't work yet)"
    namepattern = "*.py"
    pylintDir = os.path.join(cingDirTmp, 'pylint')
    #    pylintFileName = os.path.join( pylintDir, 'pylint.log')
    if os.path.exists(pylintDir):
        rmdir(pylintDir)
    mkdirs(pylintDir)
    if os.path.exists(pylintFileName):
        os.unlink(pylintFileName)
    excludedModuleList = [
        # enable exclusions for quick testing.
        #                           cingPythonDir + "/cing/core*",
        #                           cingPythonDir + "/cing/Database*",
        #                           cingPythonDir + "/cing/Libs*",
        #                           cingPythonDir + "/cing/NRG*",
        #                           cingPythonDir + "/cing/PluginCode*",
        #                           cingPythonDir + "/cing/Scripts*",
        #                           cingPythonDir + "/cing/STAR*",
        #                           cingPythonDir + "/cing/Talos*",
        # Normal set:
        cingPythonDir + "/cing/Database/CCPN*",
        cingPythonDir + "/cyana2cing*",
        cingPythonDir + "/pdbe2*",
        cingPythonDir + "/queen*",
        cingPythonDir + "/Refine*",
        cingPythonDir + "/UtilsAnalysis*",
        cingPythonDir + "/xplorcing*"
    ]
    startdir = cingPythonDir
    nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
    # enable next line(s) to do a couple of checks only.
    #    nameList = ['/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
    #                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py',
    #                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn_2.py']
    nTdebug('Will unit check: ' + repr(nameList))
    f = ForkOff(
        processes_max=cing.ncpus,
        max_time_to_wait=600,  # on a very slow setup
        verbosity=2)
    job_list = []
    for name in nameList:
        job_list.append((doPylintByName, (name, excludedModuleList)))
    done_list = f.forkoff_start(job_list, 0)
    nTmessage("Finished ids: %s", done_list)

    for name in nameList:
        mod_name = pathToModuleName(name)
        if mod_name in excludedModuleList:
            print "Skipping module:  " + mod_name
            return
        pylintOutputFileName = os.path.join(pylintDir, mod_name + '.log')
        if not os.path.exists(pylintDir):
            nTerror("Failed to find pylint output: " + pylintOutputFileName)
            continue
        if appendTextFileToFile(pylintOutputFileName, pylintFileName):
            nTerror("Failed to appendTextFileToFile")
#        nTdebug("Done appending from: %s" % pylintOutputFileName)
# end for
    doPylintOverallSummary(pylintFileName=pylintFileName)
    nTmessage("Done with pylint")
예제 #9
0
파일: main.py 프로젝트: VuisterLab/cing
def doPylintOverall(pylintFileName='pylint.txt'):
    "Add the ones you don't want to pylint (perhaps you know they don't work yet)"
    namepattern = "*.py"
    pylintDir = os.path.join( cingDirTmp, 'pylint' )
#    pylintFileName = os.path.join( pylintDir, 'pylint.log')
    if os.path.exists( pylintDir ):
        rmdir( pylintDir )
    mkdirs( pylintDir )
    if os.path.exists( pylintFileName ):
        os.unlink(pylintFileName)
    excludedModuleList = [ 
                            # enable exclusions for quick testing.
#                           cingPythonDir + "/cing/core*",
#                           cingPythonDir + "/cing/Database*",
#                           cingPythonDir + "/cing/Libs*",
#                           cingPythonDir + "/cing/NRG*",
#                           cingPythonDir + "/cing/PluginCode*",
#                           cingPythonDir + "/cing/Scripts*",
#                           cingPythonDir + "/cing/STAR*",
#                           cingPythonDir + "/cing/Talos*",
                           # Normal set:
                           cingPythonDir + "/cing/Database/CCPN*",
                           cingPythonDir + "/cyana2cing*",
                           cingPythonDir + "/pdbe2*",
                           cingPythonDir + "/queen*",
                           cingPythonDir + "/Refine*",
                           cingPythonDir + "/UtilsAnalysis*",
                           cingPythonDir + "/xplorcing*"
                          ]
    startdir = cingPythonDir
    nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
    # enable next line(s) to do a couple of checks only.
#    nameList = ['/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
#                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py',
#                '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn_2.py']
    nTdebug('Will unit check: ' + repr(nameList))
    f = ForkOff(
            processes_max=cing.ncpus,
            max_time_to_wait=600, # on a very slow setup
            verbosity=2            
            )
    job_list = []
    for name in nameList:
        job_list.append( (doPylintByName, (name, excludedModuleList)) )
    done_list = f.forkoff_start(job_list, 0)
    nTmessage("Finished ids: %s", done_list)

    for name in nameList:
        mod_name = pathToModuleName( name )
        if mod_name in excludedModuleList:
            print "Skipping module:  " + mod_name
            return
        pylintOutputFileName = os.path.join( pylintDir, mod_name + '.log')        
        if not os.path.exists( pylintDir ):
            nTerror("Failed to find pylint output: " + pylintOutputFileName)
            continue
        if appendTextFileToFile( pylintOutputFileName, pylintFileName):
            nTerror("Failed to appendTextFileToFile")
#        nTdebug("Done appending from: %s" % pylintOutputFileName)
    # end for
    doPylintOverallSummary(pylintFileName=pylintFileName)
    nTmessage("Done with pylint")
예제 #10
0
def doScriptOnEntryList(pythonScriptFileName,
          entryListFileName,
          startDir                       ='.',
          processes_max                  = 3,   # default 3
          max_time_to_wait               = 600,
          delay_between_submitting_jobs  = 1,
          extraArgList                   = None,
          start_entry_id                 = start_entry_id,
          max_entries_todo               = max_entries_todo,
          expectPdbEntryList             = True,
          shuffleBeforeSelecting         = False, # fails for chain ids when included.
          entryList                      = () # as an alternative to a file.
          ):
    """Return True on error"""
    
    if True: # DEFAULT: True
        pid = os.getpid()
        nTmessage("Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes." % pid)
        nTmessage("entryListFileName            : %s" % entryListFileName)        
        nTmessage("startDir                     : %s" % startDir                     )        
        nTmessage("processes_max                : %s" % processes_max                )        
        nTmessage("max_time_to_wait             : %s" % max_time_to_wait             )        
        nTmessage("delay_between_submitting_jobs: %s" % delay_between_submitting_jobs)        
        nTmessage("extraArgList                 : %s" % str(extraArgList            ))        
        nTmessage("start_entry_id               : %s" % start_entry_id               )        
        nTmessage("max_entries_todo             : %s" % max_entries_todo             )        
        nTmessage("expectPdbEntryList           : %s" % expectPdbEntryList           )        
        nTmessage("shuffleBeforeSelecting       : %s" % shuffleBeforeSelecting       )
    # end if        
#    if os.chdir(cingDirTmp):
#        raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)

    # Empty list means no filtering done.
    entryCodeListFilter = []
#    entryCodeListFilter = string.split("1n62")
    
    if entryListFileName:
        entryList = [line.strip() for line in open(entryListFileName)]
    entryList = [x for x in entryList if x]
    entryCountTotal = len(entryList)
    
    if expectPdbEntryList:
        chainCodeList = []
        entryCodeList = []
        for ss in entryList:
 
            entryCode = ss[:4].lower()
            if entryCode not in entryCodeListFilter:
 
                if len(ss) > 4:
                    chainCode = ss[4].upper()
                else:
                    chainCode = ''
 
                entryCodeList.append(entryCode)
                chainCodeList.append(chainCode)
          
        entryCountSelected = len(entryCodeList)
    
    else:
        #Non-PDB
        entryCodeList = [x for x in entryList if x not in entryCodeListFilter]
        entryCountSelected = len(entryCodeList)
        chainCodeList = [''] * entryCountSelected
    
    
    # lastEntryId is id of last entry excluding the entry itself.
    lastEntryId = min(len(entryCodeList), start_entry_id+max_entries_todo)    
    if shuffleBeforeSelecting:
        nTmessage("Shuffling entry list before selecting entries.")
        entryCodeListCopy = entryCodeList[:]
        shuffle(entryCodeListCopy)
        entryCodeList = entryCodeListCopy[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
    else:
        entryCodeList = entryCodeList[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
    entryCodeList.sort()
    chainCodeList = chainCodeList[start_entry_id:lastEntryId]

    nTmessage('Read      %05d entries    ' % entryCountTotal)
    nTmessage('Selected  %05d entries    ' % entryCountSelected)
    nTmessage('Sliced    %05d entries: %s' % (len(entryCodeList), entryCodeList ))
#    nTmessage('Sliced    %05d chains:  %s' % (len(chainCodeList), chainCodeList ))


    useAnyCmd = False
    if pythonScriptFileName.endswith('.py'):
        (_directory, pythonScriptFileNameRoot, _ext) = nTpath(pythonScriptFileName)
    else:
        useAnyCmd = True
        pythonScriptFileNameRoot = pythonScriptFileName.split()[0] # get first word of eg.
        if pythonScriptFileNameRoot.count('/'):
            pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('/')[-1]
        if pythonScriptFileNameRoot.count('.'):
            pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('.')[0]
        # 'cing -v 9 -n $x --initPDB $PDB/pdb$x.ent.gz --validateFastest --ranges cv')
    mkSubDirStructure( startDir, entryCodeList, pythonScriptFileNameRoot )
    logScriptFileNameRoot = 'log_'+pythonScriptFileNameRoot
    job_list = []
    for i, entry_code in enumerate(entryCodeList):
        extraArgListStr = ''
        if extraArgList:
            extraArgListStr = ' '.join( extraArgList )
        chain_code = chainCodeList[i]

        entryCodeChar2and3 = entry_code[1:3]
        entryDir = os.path.join( startDir, DATA_STR, entryCodeChar2and3, entry_code )

        date_stamp = getDateTimeStampForFileName()
        cmd = 'cd %s; python -u %s %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
            entryDir,
            pythonScriptFileName,
            entry_code,
            chain_code,
            extraArgListStr,
            logScriptFileNameRoot,
            entry_code,
            chain_code,
            date_stamp
             )
        if useAnyCmd:
            cmdBase = pythonScriptFileName
            cmdBase = cmdBase.replace("$x", entry_code)
            cmdBase = cmdBase.replace("$c", chain_code)
            cmd = 'cd %s; %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
                entryDir,
                cmdBase,
                entry_code,
                chain_code,
                logScriptFileNameRoot,
                entry_code,
                chain_code,
                date_stamp
                 )
        job = ( do_cmd, (cmd,) )
#        nTdebug("Will schedule job cmd: %s" % cmd)
        job_list.append( job )

    f = ForkOff( processes_max       = processes_max, max_time_to_wait    = max_time_to_wait)
    done_entry_list = f.forkoff_start( job_list, delay_between_submitting_jobs )
    done_entry_list.sort()
    not_done_entry_list = range(len(job_list))
    for id in done_entry_list:
        idx = not_done_entry_list.index(id)
        if idx >= 0:
            del(not_done_entry_list[idx])
    nTmessage("In doScriptOnEntryList Finished list  : %s" % done_entry_list)
    nTmessage("In doScriptOnEntryList Unfinished list: %s" % not_done_entry_list)
    for id in not_done_entry_list:
        job = job_list[id]
        _do_cmd, cmdTuple = job
        cmd = cmdTuple[0]
        nTerror("In doScriptOnEntryList failed forked: %s" % cmd)
예제 #11
0
def doFunctionOnEntryList(
          f,
          entryListFileName,
          processes_max                  = 3,   # default 3
          max_time_to_wait               = 600,
          delay_between_submitting_jobs  = 0,
          extraArgList                   = None,
          start_entry_id                 = start_entry_id,
          max_entries_todo               = max_entries_todo,
          shuffleBeforeSelecting         = False, # fails for chain ids when included.
          entryList                      = None # as an alternative to a file.
          ):
    """Return True on error"""
    
    if True: # DEFAULT: True
        pid = os.getpid()
        nTmessage("Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes." % pid)
        nTmessage("entryListFileName            : %s" % entryListFileName)        
        nTmessage("processes_max                : %s" % processes_max                )        
        nTmessage("max_time_to_wait             : %s" % max_time_to_wait             )        
        nTmessage("delay_between_submitting_jobs: %s" % delay_between_submitting_jobs)        
        nTmessage("extraArgList                 : %s" % str(extraArgList            ))        
        nTmessage("start_entry_id               : %s" % start_entry_id               )        
        nTmessage("max_entries_todo             : %s" % max_entries_todo             )        
        nTmessage("shuffleBeforeSelecting       : %s" % shuffleBeforeSelecting       )
        nTmessage("entryList                    : %s" % entryList)        
    # end if        
#    if os.chdir(cingDirTmp):
#        raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)

    # Empty list means no filtering done.
    entryCodeListFilter = []
#    entryCodeListFilter = string.split("1n62")
    if entryListFileName:
        entryListFile = file(entryListFileName, 'r')
        entryCodeList = []
        entryCountTotal = 0
        for line in entryListFile.readlines():
            line = line.strip()
            if line == '': # skip empty lines.
                continue
            entryCountTotal += 1
            entryCode = line
            if entryCode in entryCodeListFilter:
                continue
            entryCodeList.append( entryCode )
        entryListFile.close()
    else:
        entryCodeList = entryList
        entryCountTotal = len( entryList )
    # end if
    
    entryCountSelected = len( entryCodeList )
    # lastEntryId is id of last entry excluding the entry itself.
    lastEntryId = min(len(entryCodeList), start_entry_id+max_entries_todo)    
    if shuffleBeforeSelecting:
        nTmessage("Shuffling entry list before selecting entries.")
        entryCodeListCopy = entryCodeList[:]
        shuffle(entryCodeListCopy)
        entryCodeList = entryCodeListCopy[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
    else:
        entryCodeList = entryCodeList[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
    entryCodeList.sort()

    nTmessage('Read      %05d entries    ' % entryCountTotal)
    nTmessage('Selected  %05d entries    ' % entryCountSelected)
    nTmessage('Sliced    %05d entries: %s' % (len(entryCodeList), entryCodeList ))
#    nTmessage('Sliced    %05d chains:  %s' % (len(chainCodeList), chainCodeList ))

    job_list = []
    for _i, entry_code in enumerate(entryCodeList):
#        extraArgListStr = ''
#        if extraArgList:
#            extraArgListStr = ' '.join( extraArgList )
        argList = [entry_code]
        if isinstance(extraArgList, list) or isinstance(extraArgList, tuple):
            argList += extraArgList
#        else:
#            nTdebug("Skipping non-List/Tuple extraArgList: %s" % str(extraArgList))
        # end if
        job = ( f, tuple( argList ) )
        job_list.append( job )
    # end for
    f = ForkOff( processes_max       = processes_max, max_time_to_wait    = max_time_to_wait)
    done_entry_list = f.forkoff_start( job_list, delay_between_submitting_jobs )
    done_entry_list.sort()
    not_done_entry_list = range(len(job_list))
    for id in done_entry_list:
        idx = not_done_entry_list.index(id)
        if idx >= 0:
            del(not_done_entry_list[idx])
        # end if
    # end for
    nTmessage("In doScriptOnEntryList Finished list  : %s" % done_entry_list)
    nTmessage("In doScriptOnEntryList Unfinished list: %s" % not_done_entry_list)
    for id in not_done_entry_list:
        job = job_list[id]
        _do_cmd, cmdTuple = job
        cmd = cmdTuple[0]
        nTerror("In doFunctionOnEntryList failed forked: %s" % cmd)
예제 #12
0
파일: refine.py 프로젝트: jakesyl/cing
def analyze(config, project, parameters, doPrint=0 ):
    '''Analyze a run
    Returns True on failure.
    '''
    nTmessage("\n-- analyze --")

    inPath     = config.directories.converted
    modelList  = asci2list(parameters.models)
    if getDeepByKeysOrAttributes( parameters, USE_ANNEALED_STR):
        inPath     = config.directories.annealed
        modelList  = asci2list(parameters.modelsAnneal)

    nTdebug( 'inPath:         %s' % inPath)
    nTdebug( 'modelList:      %s' % modelList)

    # first create the jobs, run later
    analyzeJobs = []
    for i in modelList:
        job = Analyze(
                        config,
                        parameters,
                        project = project,
                        fileNum=i,
                        molecules=[
                                      NTdict(
                                                psfFile=parameters.psfFile,
                                                pdbFile=parameters.baseName % i,
                                                selectionCode='(not resn TIP3 and not resn ANI)'
                                  ),
                         ],
                         inPath     = inPath,
                         outPath    = config.directories.analyzed,
                        jobName='analyze_%d' % i,
                     )

        analyzeJobs.append(job)
    #end for


    job_list = []
    for job in analyzeJobs:
        if job.createScript():
            nTerror("In refine#analyze failed to create at least one job's script.")
            return True
        if doPrint:
            job.printScript()
        #end if
#        job.runScript()
        job_list.append((job.runScript,) )
    #end for
    f = ForkOff(
            processes_max=cing.ncpus,
            max_time_to_wait=600,
            verbosity=cing.verbosity
        )
    done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
    nTmessage("Finished ids: %s", done_list)
    if not done_list:
        notDone = parameters.modelCountAnneal - len(done_list)
        nTerror("Failed to analyze %s", notDone)
        return True
예제 #13
0
def doScriptOnEntryList(
    pythonScriptFileName,
    entryListFileName,
    startDir='.',
    processes_max=3,  # default 3
    max_time_to_wait=600,
    delay_between_submitting_jobs=1,
    extraArgList=None,
    start_entry_id=start_entry_id,
    max_entries_todo=max_entries_todo,
    expectPdbEntryList=True,
    shuffleBeforeSelecting=False,  # fails for chain ids when included.
    entryList=()  # as an alternative to a file.
):
    """Return True on error"""

    if True:  # DEFAULT: True
        pid = os.getpid()
        nTmessage(
            "Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes."
            % pid)
        nTmessage("entryListFileName            : %s" % entryListFileName)
        nTmessage("startDir                     : %s" % startDir)
        nTmessage("processes_max                : %s" % processes_max)
        nTmessage("max_time_to_wait             : %s" % max_time_to_wait)
        nTmessage("delay_between_submitting_jobs: %s" %
                  delay_between_submitting_jobs)
        nTmessage("extraArgList                 : %s" % str(extraArgList))
        nTmessage("start_entry_id               : %s" % start_entry_id)
        nTmessage("max_entries_todo             : %s" % max_entries_todo)
        nTmessage("expectPdbEntryList           : %s" % expectPdbEntryList)
        nTmessage("shuffleBeforeSelecting       : %s" % shuffleBeforeSelecting)
    # end if


#    if os.chdir(cingDirTmp):
#        raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)

# Empty list means no filtering done.
    entryCodeListFilter = []
    #    entryCodeListFilter = string.split("1n62")

    if entryListFileName:
        entryList = [line.strip() for line in open(entryListFileName)]
    entryList = [x for x in entryList if x]
    entryCountTotal = len(entryList)

    if expectPdbEntryList:
        chainCodeList = []
        entryCodeList = []
        for ss in entryList:

            entryCode = ss[:4].lower()
            if entryCode not in entryCodeListFilter:

                if len(ss) > 4:
                    chainCode = ss[4].upper()
                else:
                    chainCode = ''

                entryCodeList.append(entryCode)
                chainCodeList.append(chainCode)

        entryCountSelected = len(entryCodeList)

    else:
        #Non-PDB
        entryCodeList = [x for x in entryList if x not in entryCodeListFilter]
        entryCountSelected = len(entryCodeList)
        chainCodeList = [''] * entryCountSelected

    # lastEntryId is id of last entry excluding the entry itself.
    lastEntryId = min(len(entryCodeList), start_entry_id + max_entries_todo)
    if shuffleBeforeSelecting:
        nTmessage("Shuffling entry list before selecting entries.")
        entryCodeListCopy = entryCodeList[:]
        shuffle(entryCodeListCopy)
        entryCodeList = entryCodeListCopy[
            start_entry_id:
            lastEntryId]  # no sense in starting at zero here; they're random.
    else:
        entryCodeList = entryCodeList[
            start_entry_id:
            lastEntryId]  # no sense in starting at zero here; they're random.
    entryCodeList.sort()
    chainCodeList = chainCodeList[start_entry_id:lastEntryId]

    nTmessage('Read      %05d entries    ' % entryCountTotal)
    nTmessage('Selected  %05d entries    ' % entryCountSelected)
    nTmessage('Sliced    %05d entries: %s' %
              (len(entryCodeList), entryCodeList))
    #    nTmessage('Sliced    %05d chains:  %s' % (len(chainCodeList), chainCodeList ))

    useAnyCmd = False
    if pythonScriptFileName.endswith('.py'):
        (_directory, pythonScriptFileNameRoot,
         _ext) = nTpath(pythonScriptFileName)
    else:
        useAnyCmd = True
        pythonScriptFileNameRoot = pythonScriptFileName.split()[
            0]  # get first word of eg.
        if pythonScriptFileNameRoot.count('/'):
            pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('/')[-1]
        if pythonScriptFileNameRoot.count('.'):
            pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('.')[0]
        # 'cing -v 9 -n $x --initPDB $PDB/pdb$x.ent.gz --validateFastest --ranges cv')
    mkSubDirStructure(startDir, entryCodeList, pythonScriptFileNameRoot)
    logScriptFileNameRoot = 'log_' + pythonScriptFileNameRoot
    job_list = []
    for i, entry_code in enumerate(entryCodeList):
        extraArgListStr = ''
        if extraArgList:
            extraArgListStr = ' '.join(extraArgList)
        chain_code = chainCodeList[i]

        entryCodeChar2and3 = entry_code[1:3]
        entryDir = os.path.join(startDir, DATA_STR, entryCodeChar2and3,
                                entry_code)

        date_stamp = getDateTimeStampForFileName()
        cmd = 'cd %s; python -u %s %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
            entryDir, pythonScriptFileName, entry_code, chain_code,
            extraArgListStr, logScriptFileNameRoot, entry_code, chain_code,
            date_stamp)
        if useAnyCmd:
            cmdBase = pythonScriptFileName
            cmdBase = cmdBase.replace("$x", entry_code)
            cmdBase = cmdBase.replace("$c", chain_code)
            cmd = 'cd %s; %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
                entryDir, cmdBase, entry_code, chain_code,
                logScriptFileNameRoot, entry_code, chain_code, date_stamp)
        job = (do_cmd, (cmd, ))
        #        nTdebug("Will schedule job cmd: %s" % cmd)
        job_list.append(job)

    f = ForkOff(processes_max=processes_max, max_time_to_wait=max_time_to_wait)
    done_entry_list = f.forkoff_start(job_list, delay_between_submitting_jobs)
    done_entry_list.sort()
    not_done_entry_list = range(len(job_list))
    for id in done_entry_list:
        idx = not_done_entry_list.index(id)
        if idx >= 0:
            del (not_done_entry_list[idx])
    nTmessage("In doScriptOnEntryList Finished list  : %s" % done_entry_list)
    nTmessage("In doScriptOnEntryList Unfinished list: %s" %
              not_done_entry_list)
    for id in not_done_entry_list:
        job = job_list[id]
        _do_cmd, cmdTuple = job
        cmd = cmdTuple[0]
        nTerror("In doScriptOnEntryList failed forked: %s" % cmd)
예제 #14
0
def doFunctionOnEntryList(
    f,
    entryListFileName,
    processes_max=3,  # default 3
    max_time_to_wait=600,
    delay_between_submitting_jobs=0,
    extraArgList=None,
    start_entry_id=start_entry_id,
    max_entries_todo=max_entries_todo,
    shuffleBeforeSelecting=False,  # fails for chain ids when included.
    entryList=None  # as an alternative to a file.
):
    """Return True on error"""

    if True:  # DEFAULT: True
        pid = os.getpid()
        nTmessage(
            "Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes."
            % pid)
        nTmessage("entryListFileName            : %s" % entryListFileName)
        nTmessage("processes_max                : %s" % processes_max)
        nTmessage("max_time_to_wait             : %s" % max_time_to_wait)
        nTmessage("delay_between_submitting_jobs: %s" %
                  delay_between_submitting_jobs)
        nTmessage("extraArgList                 : %s" % str(extraArgList))
        nTmessage("start_entry_id               : %s" % start_entry_id)
        nTmessage("max_entries_todo             : %s" % max_entries_todo)
        nTmessage("shuffleBeforeSelecting       : %s" % shuffleBeforeSelecting)
        nTmessage("entryList                    : %s" % entryList)
    # end if
#    if os.chdir(cingDirTmp):
#        raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)

# Empty list means no filtering done.
    entryCodeListFilter = []
    #    entryCodeListFilter = string.split("1n62")
    if entryListFileName:
        entryListFile = file(entryListFileName, 'r')
        entryCodeList = []
        entryCountTotal = 0
        for line in entryListFile.readlines():
            line = line.strip()
            if line == '':  # skip empty lines.
                continue
            entryCountTotal += 1
            entryCode = line
            if entryCode in entryCodeListFilter:
                continue
            entryCodeList.append(entryCode)
        entryListFile.close()
    else:
        entryCodeList = entryList
        entryCountTotal = len(entryList)
    # end if

    entryCountSelected = len(entryCodeList)
    # lastEntryId is id of last entry excluding the entry itself.
    lastEntryId = min(len(entryCodeList), start_entry_id + max_entries_todo)
    if shuffleBeforeSelecting:
        nTmessage("Shuffling entry list before selecting entries.")
        entryCodeListCopy = entryCodeList[:]
        shuffle(entryCodeListCopy)
        entryCodeList = entryCodeListCopy[
            start_entry_id:
            lastEntryId]  # no sense in starting at zero here; they're random.
    else:
        entryCodeList = entryCodeList[
            start_entry_id:
            lastEntryId]  # no sense in starting at zero here; they're random.
    entryCodeList.sort()

    nTmessage('Read      %05d entries    ' % entryCountTotal)
    nTmessage('Selected  %05d entries    ' % entryCountSelected)
    nTmessage('Sliced    %05d entries: %s' %
              (len(entryCodeList), entryCodeList))
    #    nTmessage('Sliced    %05d chains:  %s' % (len(chainCodeList), chainCodeList ))

    job_list = []
    for _i, entry_code in enumerate(entryCodeList):
        #        extraArgListStr = ''
        #        if extraArgList:
        #            extraArgListStr = ' '.join( extraArgList )
        argList = [entry_code]
        if isinstance(extraArgList, list) or isinstance(extraArgList, tuple):
            argList += extraArgList
#        else:
#            nTdebug("Skipping non-List/Tuple extraArgList: %s" % str(extraArgList))
# end if
        job = (f, tuple(argList))
        job_list.append(job)
    # end for
    f = ForkOff(processes_max=processes_max, max_time_to_wait=max_time_to_wait)
    done_entry_list = f.forkoff_start(job_list, delay_between_submitting_jobs)
    done_entry_list.sort()
    not_done_entry_list = range(len(job_list))
    for id in done_entry_list:
        idx = not_done_entry_list.index(id)
        if idx >= 0:
            del (not_done_entry_list[idx])
        # end if
    # end for
    nTmessage("In doScriptOnEntryList Finished list  : %s" % done_entry_list)
    nTmessage("In doScriptOnEntryList Unfinished list: %s" %
              not_done_entry_list)
    for id in not_done_entry_list:
        job = job_list[id]
        _do_cmd, cmdTuple = job
        cmd = cmdTuple[0]
        nTerror("In doFunctionOnEntryList failed forked: %s" % cmd)