def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = False # default: False htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doQueeny = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. # NB leave this set to True or modify code below. removeCcpnDirectory = 1 # perhaps not so in the future. modelCount = None # default setting is None # ranges = None if fastestTest: modelCount = 2 # if this is more and there is only one model present it leads to an error message. htmlOnly = True doWhatif = False doProcheck = False doWattos = False doQueeny = False doTalos = False forceRedo = True forceRetrieveInput = True nTmessage(header) nTmessage(getStartMessage()) # Sync below code with nrgCing#createToposTokens expectedArgumentList = """ verbosity inputDir outputDir pdbConvention restraintsConvention archiveType projectType storeCING2db ranges filterTopViolations filterVasco singleCoreOperation """.split() expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTmessage("consider updating code to include all sequential parameters: %s" % str(expectedArgumentList)) if len(extraArgList) > expectedNumberOfArguments: nTerror("Got arguments: " + str(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % ( expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True # end if # end if entryCodeChar2and3 = entryId[1:3] cing.verbosity = int( extraArgList[IDX_VERBOSITY] ) inputDir = extraArgList[IDX_INPUT] outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable restraintsConvention = extraArgList[IDX_RESTRAINTS] archiveType = extraArgList[IDX_ARCHIVE] # Only used for deriving the input location not the output. projectType = extraArgList[IDX_PROJECT_TYPE] storeCING2db = stringMeansBooleanTrue( getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB)) ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES) filterTopViolations = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_TOP) if filterTopViolations: filterTopViolations = int(filterTopViolations) # change '0' to 0 filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO) if filterVasco: filterVasco = int(filterVasco) else: filterVasco = 1 # Default should be True # end if singleCoreOperation = getDeepByKeysOrAttributes(extraArgList, IDX_SINGLE_CORE_OPERATION ) if singleCoreOperation: singleCoreOperation = int(singleCoreOperation) else: singleCoreOperation = 0 # Default should be True # end if if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) isRemoteOutputDir = False if '@' in outputDir: isRemoteOutputDir = True # end if # vc = vCing('.') # argument is a fake master_ssh_url not needed here. archive_id = getArchiveIdFromDirectoryName( outputDir ) nTdebug("Using program arguments:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) nTdebug("filterTopViolations: %s" % filterTopViolations) nTdebug("filterVasco: %s" % filterVasco) nTdebug("singleCoreOperation: %s" % singleCoreOperation) nTdebug("") nTdebug("Using derived settings:") nTdebug("modelCount: %s" % modelCount) nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir) nTdebug("archive_id: %s" % archive_id) # For NMR_REDO required as most efficient. if singleCoreOperation: setToSingleCoreOperation() # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if forceRedo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. if isRemoteOutputDir: os.chdir(cingDirTmp) else: os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. formatFileName = '%s.tgz' if projectType == PROJECT_TYPE_CING: formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) allowedInputProtocolList = 'http file ssh'.split() inputProtocal = string.split( inputDir, ':' )[0] if inputProtocal in allowedInputProtocolList: stillToRetrieve = False if os.path.exists(fileNameTgz): if forceRetrieveInput: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if else: nTdebug("Entry not retrieved which might be normal in some situations.") # end if. if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. # fullFileNameTgz = os.path.join(inputDir, fileNameTgz) # shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') pdbFilePath = entryId + ".pdb" gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # end if if ranges is not None: project.molecule.setRanges(ranges) # end if if archive_id: project.molecule.setArchiveId(archive_id) # end if project.molecule.superpose(ranges=ranges) if filterTopViolations and not project.filterHighRestraintViol(): nTerror("Failed to filterHighRestraintViol") ####> MAIN UTILITY HERE if 0: # DEFAULT 0 project.save() if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doQueeny = doQueeny, doTalos=doTalos, filterVasco = filterVasco ): nTerror("Failed to validate project read") return True # end if filterVasco # Write a single PDB file containing all models # according to IUPAC conventions project.export2PDB() project.save() if storeCING2db and archive_id: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. # archive_id = ARCHIVE_DEV_NRG_ID # if isProduction: # archive_id = ARCHIVE_NRG_ID try: if doStoreCING2db( entryId, archive_id, project=project): nTerror("Failed to store CING project's data to DB but continuing.") except: nTtracebackError() nTerror("Failed to store CING project's data due to above traceback error.") if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' os.unlink(fileNameTgz) # temporary ccpn tgz if removeCcpnDirectory: rmdir(entryId) # ccpn dir may contain vasco info. if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) nTdebug("cmd: %s" % cmd) # do_cmd(cmd) status, result = commands.getstatusoutput(cmd) if status: nTerror("Failed to tar status: %s with result %s" % (status, result)) return True if isRemoteOutputDir: if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax = 2): nTerror("Failed to send File By Scp status: %s with result %s" % (status, result)) nTerror("Maintaining results.") return True # end if nTmessage("Removing tgz result: %s" % tgzFileNameCing) os.remove(tgzFileNameCing) nTmessage("Removing cing dir itself: %s" % directoryNameCing) rmdir(directoryNameCing) else: # do NOT remove local copy pass
def test_ccpn(self): # cing.verbosity = verbosityDebug # if you have a local copy you can use it; make sure to adjust the path setting below. fastestTest = True # DEFAULT: True. Not passed to the validate routine in order to customize checks for speed. modelCount = 99 # DEFAULT: 99 redoFromCingProject = False # DEFAULT: False htmlOnly = False # DEFAULT: False # default is False but enable it for faster runs without some actual data. doWhatif = True # DEFAULT: True # disables whatif actual run doProcheck = True # DEFAULT: True doWattos = True # DEFAULT: True doQueeny = True # DEFAULT: True doTalos = True # DEFAULT: True filterVasco = True # DEFAULT: True filterTopViolations = True # DEFAULT: True useNrgArchive = False # DEFAULT: False ranges = CV_STR # DEFAULT: CV_STR # ranges='173-177' # ranges='6-13,29-45' # 1bus doSwapCheck = False doRestoreCheck = False doStoreCheck = False # DEFAULT: False Requires sqlAlchemy doSave = not redoFromCingProject # DEFAULT: False Requires sqlAlchemy if fastestTest: modelCount = 2 # DEFAULT 2 # redoFromCingProject = False htmlOnly = True # DEFAULT: True doWhatif = False # DEFAULT: False doProcheck = False # DEFAULT: False doWattos = False # DEFAULT: False doQueeny = False # DEFAULT: False doTalos = False # DEFAULT: False filterVasco = False # DEFAULT: False doRestoreCheck = False # DEFAULT: False doStoreCheck = False # DEFAULT: False if redoFromCingProject: useNrgArchive = False doWhatif = False doProcheck = False doWattos = False doTalos = False cingDirTmpTest = os.path.join(cingDirTmp, getCallerName()) mkdirs(cingDirTmpTest) self.failIf(os.chdir(cingDirTmpTest), msg="Failed to change to test directory for files: " + cingDirTmpTest) for i, entryId in enumerate(AllChecks.entryList): if i: nTmessage('\n\n') if redoFromCingProject: project = Project.open(entryId, status='old') else: project = Project.open(entryId, status='new') self.assertTrue(project, 'Failed opening project: ' + entryId) if useNrgArchive: # default is False # inputArchiveDir = os.path.join('/Library/WebServer/Documents/NRG-CING/recoordSync', entryId) # Mounted from nmr.cmbi.ru.nl # inputArchiveDir = os.path.join('/Volumes/tria1/Library/WebServer/Documents/NRG-CING/recoordSync', entryId) inputArchiveDir = os.path.join( '/Users/jd/ccpn_tmp/data/recoord', entryId) else: inputArchiveDir = os.path.join(cingDirTestsData, "ccpn") ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz") if not os.path.exists(ccpnFile): ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz") if not os.path.exists(ccpnFile): self.fail("Neither %s or the .tgz exist" % ccpnFile) if doSwapCheck: # Need to start with ccpn without loading into CING api yet. if os.path.exists(entryId): nTmessage("Removing previous directory: %s" % entryId) rmtree(entryId) do_cmd("tar -xzf " + ccpnFile) # will extract to local dir. if os.path.exists('linkNmrStarData'): nTmessage( "Renaming standard directory linkNmrStarData to entry: %s" % entryId) os.rename('linkNmrStarData', entryId) ccpnProject = loadProject(entryId) if not ccpnProject: self.fail("Failed to read project: %s" % entryId) # constraintsHandler = ConstraintsHandler() nmrConstraintStore = ccpnProject.findFirstNmrConstraintStore( ) structureEnsemble = ccpnProject.findFirstStructureEnsemble( ) numSwapCheckRuns = 2 if nmrConstraintStore: if structureEnsemble: swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns) else: nTmessage( "Failed to find structureEnsemble; skipping swapCheck" ) else: nTmessage( "Failed to find nmrConstraintStore; skipping swapCheck" ) # constraintsHandler.swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns) nTmessage('Saving to new path') # checkValid=True, saveProject(ccpnProject, newPath=entryId, removeExisting=True) ccpnFile = entryId # set to local dir now. # end if doSwapCheck self.assertTrue( project.initCcpn(ccpnFolder=ccpnFile, modelCount=modelCount)) # if doSave: # self.assertTrue(project.save()) if False: ranges = "173-183" # residueOfInterest = range(171,174) # for residue in project.molecule.A.allResidues(): # if residue.resNum not in residueOfInterest: # # nTmessage("Removing residue of no interest") # project.molecule.A.removeResidue(residue) if False: ccpnProject = project.ccpn printSequenceFromCcpnProject(ccpnProject) if True: self.assertFalse(project.molecule.setRanges(ranges)) nTdebug('In test_ccpn.py: ranges: %s' % str(project.molecule.ranges)) project.molecule.rangesToMmCifRanges(ranges) if True: self.assertFalse( project.validate( htmlOnly=htmlOnly, ranges=ranges, # fastestTest=fastestTest, # disabled doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doQueeny=doQueeny, doTalos=doTalos, filterVasco=filterVasco, filterTopViolations=filterTopViolations)) if doWattos and False: mol = project.molecule completenessMol = mol.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR) nTdebug("completenessMol: %s" % completenessMol) for res in mol.allResidues(): completenessRes = res.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR) nTdebug("%s: %s" % (res, completenessRes)) # end for # end if color = project.molecule.getRogColor() nTdebug("The color of this molecule is: %s" % color) self.assertFalse(color == COLOR_RED) # ;-) # end if # self.assertTrue(project.exportValidation2ccpn()) # self.assertFalse(project.removeCcpnReferences()) # Do not leave the old CCPN directory laying around since it might get added to by another test. # if os.path.exists(entryId): # self.assertFalse(shutil.rmtree(entryId)) if False: self.assertTrue(project.save()) self.assertTrue(project.saveCcpn(entryId)) if doRestoreCheck: del project project = Project.open(entryId, status='old') self.assertTrue(project, 'Failed reopening project: ' + entryId) if doStoreCheck: # # Does require below import which is used here to trigger import warning in case it's not installed. # pylint: disable=W0612 from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0404 if doStoreCING2db(entryId, ARCHIVE_NRG_ID, project=project): nTerror( "Failed to store CING project's data to DB but continuing." ) if doSave: self.assertTrue(project.save())
def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = True # default: False # ranges=AUTO_STR # default is None retrieved from DBMS csv files. htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. modelCount = None # default setting is None if fastestTest: modelCount = 3 htmlOnly = True doWhatif = False doProcheck = False doWattos = False doTalos = False force_redo = True force_retrieve_input = True nTmessage(cing.cingDefinitions.getHeaderString()) nTmessage(cing.systemDefinitions.getStartMessage()) expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType','projectType','storeCING2db'] expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTerror("Got arguments: " + repr(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % ( expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True entryCodeChar2and3 = entryId[1:3] inputDir = extraArgList[0] outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[2] #@UnusedVariable restraintsConvention = extraArgList[3] archiveType = extraArgList[4] projectType = extraArgList[5] storeCING2db = False if len(extraArgList) >= expectedNumberOfArguments: storeCING2db = extraArgList[6] if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) ranges = None # targetId = getTargetForFullEntryName(entryId) # if not targetId: # nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId) # return True # ranges = getRangesForTarget(targetId) # if ranges == None: # nTerror("Failed to getRangesForTarget for targetId: %s" % targetId) # return True nTdebug("Using:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("modelCount: %s" % modelCount) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if force_redo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. # extension = '.tgz' formatFileName = '%s.tgz' # fileNameTgz = entryId + '.tgz' if projectType == PROJECT_TYPE_CING: # fileNameTgz = entryId + '.cing.tgz' formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) # if true will do retrieveTgzFromUrl. if inputDir.startswith("http") or inputDir.startswith("file"): stillToRetrieve = False if os.path.exists(fileNameTgz): if force_retrieve_input: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if # end if. # retrieveTgzFromUrl(entryId, inputDir) if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. fullFileNameTgz = os.path.join(inputDir, fileNameTgz) shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') # pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ] # for pdbFileName in pdbFileFormats: # pdbFileName = "pdb" + entryId + ".ent.gz" # # pdbFilePath = os.path.join( inputDir, pdbFileName) # pdbFilePath = os.path.join(inputDir, pdbFileName) # if os.path.exists(pdbFilePath): # break # tmpPdbFile = None # if pdbFileName.endswith('.gz'): pdbFilePath = entryId + ".pdb" # tmpPdbFile = pdbFilePath # if os.path.exists(pdbFilePath): # os.unlink(pdbFilePath) gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # if inputDirOrg == inputDirCASD_NMR: # if True: # Default is False for this is specific to CASD-NMR # nTmessage("Renaming molecule name to entry id: %s" % entryId) # project.molecule.name = entryId # insufficient since all data is already initialized to disk. # project.molecule.rename( entryId ) # project.save() # project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly. project.molecule.superpose(ranges=ranges) if True: if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doTalos=doTalos): nTerror("Failed to validate project read") return True if storeCING2db: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. try: if doStoreCING2db( entryId, ARCHIVE_CASP_ID, project=project): nTerror("Failed to store CING project's data to DB but continuing.") except: nTtracebackError() nTerror("Failed to store CING project's data due to above traceback error.") project.save() if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' # os.unlink(fileNameTgz) # temporary ccpn tgz rmdir(entryId) # temporary ccpn dir if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) do_cmd(cmd)
def test_queeny(self): runQueeny = 1 # DEFAULT: 1 doStoreCheck = False # DEFAULT: False Requires sqlAlchemy doValueCheck = 1 # DEFAULT: 1 Requires 1brv entryList = "1brv_cs_pk_2mdl".split( ) # DEFAULT because it contains many data types and is small/fast to run. # entryList = "1a24 1a4d 1afp 1ai0 1b4y 1brv 1brv 1bus 1cjg 1hue 1ieh 1iv6 1kr8 2cka 2hgh 2jzn 2k0e 8psh".split() expectedInfoList = [ (158, 0.0), (159, 0.0), (160, 0.0), (161, 0.0), (162, 0.0), (163, 0.0), (164, 0.0), (165, 0.000141), (166, 0.006669), (167, 0.047112), (168, 0.159379), (169, 0.309577), (170, 0.578983), ( 171, 0.838257 ), # First residue with restraints. Previously none zero elements are due to window averaging JFD assumes. (172, 1.029047), (173, 1.063531), (174, 0.969195), (175, 1.027744), (176, 0.954469), (177, 0.794625), (178, 0.822723), (179, 0.845598), (180, 0.828139), (181, 0.971881), (182, 1.141096), (183, 1.063613), (184, 0.974891), (185, 1.116405), (186, 1.113886), (187, 0.945552), (188, 0.731124), (189, 0.411986), ] cingDirTmpTest = os.path.join(cingDirTmp, getCallerName()) mkdirs(cingDirTmpTest) self.failIf(os.chdir(cingDirTmpTest), msg="Failed to change to test directory for files: " + cingDirTmpTest) for entryId in entryList: project = Project.open(entryId, status='new') self.assertTrue(project, 'Failed opening project: ' + entryId) inputArchiveDir = os.path.join(cingDirTestsData, "ccpn") ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz") if not os.path.exists(ccpnFile): ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz") if not os.path.exists(ccpnFile): self.fail("Neither %s or the .tgz exist" % ccpnFile) self.assertTrue(project.initCcpn(ccpnFolder=ccpnFile, modelCount=1)) if runQueeny: self.assertFalse(project.runQueeny()) if doValueCheck: for i, res in enumerate(project.molecule.allResidues()): expected = expectedInfoList[i][1] information = getDeepByKeysOrAttributes( res, QUEENY_INFORMATION_STR) uncertainty1 = getDeepByKeysOrAttributes( res, QUEENY_UNCERTAINTY1_STR) uncertainty2 = getDeepByKeysOrAttributes( res, QUEENY_UNCERTAINTY2_STR) nTdebug( "Comparing expected versus inf/unc1/unc2 for %20s with %8.3f %8.3f %8.3f %8.3f" % (res, expected, information, uncertainty1, uncertainty2)) # self.assertAlmostEqual(expected,information,3) if doStoreCheck: # # Does require: try: from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0612,W0404 pdbEntryId = entryId[0:4] if doStoreCING2db(pdbEntryId, ARCHIVE_DEV_NRG_ID, project=project): nTerror( "Failed to store CING project's data to DB but continuing." ) except: nTwarning("Failed to doStoreCING2db from test_queeny.py")
def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = False # default: False htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doQueeny = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. # NB leave this set to True or modify code below. removeCcpnDirectory = 1 # perhaps not so in the future. modelCount = None # default setting is None # ranges = None if fastestTest: modelCount = 2 # if this is more and there is only one model present it leads to an error message. htmlOnly = True doWhatif = False doProcheck = False doWattos = False doQueeny = False doTalos = False forceRedo = True forceRetrieveInput = True nTmessage(header) nTmessage(getStartMessage()) # Sync below code with nrgCing#createToposTokens expectedArgumentList = """ verbosity inputDir outputDir pdbConvention restraintsConvention archiveType projectType storeCING2db ranges filterTopViolations filterVasco singleCoreOperation """.split() expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTmessage( "consider updating code to include all sequential parameters: %s" % str(expectedArgumentList)) if len(extraArgList) > expectedNumberOfArguments: nTerror("Got arguments: " + str(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % (expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True # end if # end if entryCodeChar2and3 = entryId[1:3] cing.verbosity = int(extraArgList[IDX_VERBOSITY]) inputDir = extraArgList[IDX_INPUT] outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable restraintsConvention = extraArgList[IDX_RESTRAINTS] archiveType = extraArgList[ IDX_ARCHIVE] # Only used for deriving the input location not the output. projectType = extraArgList[IDX_PROJECT_TYPE] storeCING2db = stringMeansBooleanTrue( getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB)) ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES) filterTopViolations = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_TOP) if filterTopViolations: filterTopViolations = int(filterTopViolations) # change '0' to 0 filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO) if filterVasco: filterVasco = int(filterVasco) else: filterVasco = 1 # Default should be True # end if singleCoreOperation = getDeepByKeysOrAttributes(extraArgList, IDX_SINGLE_CORE_OPERATION) if singleCoreOperation: singleCoreOperation = int(singleCoreOperation) else: singleCoreOperation = 0 # Default should be True # end if if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) isRemoteOutputDir = False if '@' in outputDir: isRemoteOutputDir = True # end if # vc = vCing('.') # argument is a fake master_ssh_url not needed here. archive_id = getArchiveIdFromDirectoryName(outputDir) nTdebug("Using program arguments:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) nTdebug("filterTopViolations: %s" % filterTopViolations) nTdebug("filterVasco: %s" % filterVasco) nTdebug("singleCoreOperation: %s" % singleCoreOperation) nTdebug("") nTdebug("Using derived settings:") nTdebug("modelCount: %s" % modelCount) nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir) nTdebug("archive_id: %s" % archive_id) # For NMR_REDO required as most efficient. if singleCoreOperation: setToSingleCoreOperation() # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if forceRedo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. if isRemoteOutputDir: os.chdir(cingDirTmp) else: os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. formatFileName = '%s.tgz' if projectType == PROJECT_TYPE_CING: formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) allowedInputProtocolList = 'http file ssh'.split() inputProtocal = string.split(inputDir, ':')[0] if inputProtocal in allowedInputProtocolList: stillToRetrieve = False if os.path.exists(fileNameTgz): if forceRetrieveInput: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if else: nTdebug( "Entry not retrieved which might be normal in some situations.") # end if. if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. # fullFileNameTgz = os.path.join(inputDir, fileNameTgz) # shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') pdbFilePath = entryId + ".pdb" gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # end if if ranges is not None: project.molecule.setRanges(ranges) # end if if archive_id: project.molecule.setArchiveId(archive_id) # end if project.molecule.superpose(ranges=ranges) if filterTopViolations and not project.filterHighRestraintViol(): nTerror("Failed to filterHighRestraintViol") ####> MAIN UTILITY HERE if 0: # DEFAULT 0 project.save() if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doQueeny=doQueeny, doTalos=doTalos, filterVasco=filterVasco): nTerror("Failed to validate project read") return True # end if filterVasco # Write a single PDB file containing all models # according to IUPAC conventions project.export2PDB() project.save() if storeCING2db and archive_id: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. # archive_id = ARCHIVE_DEV_NRG_ID # if isProduction: # archive_id = ARCHIVE_NRG_ID try: if doStoreCING2db(entryId, archive_id, project=project): nTerror( "Failed to store CING project's data to DB but continuing." ) except: nTtracebackError() nTerror( "Failed to store CING project's data due to above traceback error." ) if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' os.unlink(fileNameTgz) # temporary ccpn tgz if removeCcpnDirectory: rmdir(entryId) # ccpn dir may contain vasco info. if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) nTdebug("cmd: %s" % cmd) # do_cmd(cmd) status, result = commands.getstatusoutput(cmd) if status: nTerror("Failed to tar status: %s with result %s" % (status, result)) return True if isRemoteOutputDir: if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax=2): nTerror( "Failed to send File By Scp status: %s with result %s" % (status, result)) nTerror("Maintaining results.") return True # end if nTmessage("Removing tgz result: %s" % tgzFileNameCing) os.remove(tgzFileNameCing) nTmessage("Removing cing dir itself: %s" % directoryNameCing) rmdir(directoryNameCing) else: # do NOT remove local copy pass
def test_ccpn(self): # cing.verbosity = verbosityDebug # if you have a local copy you can use it; make sure to adjust the path setting below. fastestTest = True # DEFAULT: True. Not passed to the validate routine in order to customize checks for speed. modelCount=99 # DEFAULT: 99 redoFromCingProject = False # DEFAULT: False htmlOnly = False # DEFAULT: False # default is False but enable it for faster runs without some actual data. doWhatif = True # DEFAULT: True # disables whatif actual run doProcheck = True # DEFAULT: True doWattos = True # DEFAULT: True doQueeny = True # DEFAULT: True doTalos = True # DEFAULT: True filterVasco = True # DEFAULT: True filterTopViolations = True # DEFAULT: True useNrgArchive = False # DEFAULT: False ranges = CV_STR # DEFAULT: CV_STR # ranges='173-177' # ranges='6-13,29-45' # 1bus doSwapCheck = False doRestoreCheck = False doStoreCheck = False # DEFAULT: False Requires sqlAlchemy doSave = not redoFromCingProject # DEFAULT: False Requires sqlAlchemy if fastestTest: modelCount=2 # DEFAULT 2 # redoFromCingProject = False htmlOnly = True # DEFAULT: True doWhatif = False # DEFAULT: False doProcheck = False # DEFAULT: False doWattos = False # DEFAULT: False doQueeny = False # DEFAULT: False doTalos = False # DEFAULT: False filterVasco = False # DEFAULT: False doRestoreCheck = False # DEFAULT: False doStoreCheck = False # DEFAULT: False if redoFromCingProject: useNrgArchive = False doWhatif = False doProcheck = False doWattos = False doTalos = False cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() ) mkdirs( cingDirTmpTest ) self.failIf(os.chdir(cingDirTmpTest), msg = "Failed to change to test directory for files: " + cingDirTmpTest) for i,entryId in enumerate(AllChecks.entryList): if i: nTmessage('\n\n') if redoFromCingProject: project = Project.open(entryId, status = 'old') else: project = Project.open(entryId, status = 'new') self.assertTrue(project, 'Failed opening project: ' + entryId) if useNrgArchive: # default is False # inputArchiveDir = os.path.join('/Library/WebServer/Documents/NRG-CING/recoordSync', entryId) # Mounted from nmr.cmbi.ru.nl # inputArchiveDir = os.path.join('/Volumes/tria1/Library/WebServer/Documents/NRG-CING/recoordSync', entryId) inputArchiveDir = os.path.join('/Users/jd/ccpn_tmp/data/recoord', entryId) else: inputArchiveDir = os.path.join(cingDirTestsData, "ccpn") ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz") if not os.path.exists(ccpnFile): ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz") if not os.path.exists(ccpnFile): self.fail("Neither %s or the .tgz exist" % ccpnFile) if doSwapCheck: # Need to start with ccpn without loading into CING api yet. if os.path.exists(entryId): nTmessage("Removing previous directory: %s" % entryId) rmtree(entryId) do_cmd("tar -xzf " + ccpnFile) # will extract to local dir. if os.path.exists('linkNmrStarData'): nTmessage("Renaming standard directory linkNmrStarData to entry: %s" % entryId) os.rename('linkNmrStarData', entryId) ccpnProject = loadProject(entryId) if not ccpnProject: self.fail("Failed to read project: %s" % entryId) # constraintsHandler = ConstraintsHandler() nmrConstraintStore = ccpnProject.findFirstNmrConstraintStore() structureEnsemble = ccpnProject.findFirstStructureEnsemble() numSwapCheckRuns = 2 if nmrConstraintStore: if structureEnsemble: swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns) else: nTmessage("Failed to find structureEnsemble; skipping swapCheck") else: nTmessage("Failed to find nmrConstraintStore; skipping swapCheck") # constraintsHandler.swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns) nTmessage('Saving to new path') # checkValid=True, saveProject(ccpnProject, newPath=entryId, removeExisting=True) ccpnFile = entryId # set to local dir now. # end if doSwapCheck self.assertTrue(project.initCcpn(ccpnFolder = ccpnFile, modelCount=modelCount)) # if doSave: # self.assertTrue(project.save()) if False: ranges = "173-183" # residueOfInterest = range(171,174) # for residue in project.molecule.A.allResidues(): # if residue.resNum not in residueOfInterest: # # nTmessage("Removing residue of no interest") # project.molecule.A.removeResidue(residue) if False: ccpnProject = project.ccpn printSequenceFromCcpnProject(ccpnProject) if True: self.assertFalse(project.molecule.setRanges(ranges)) nTdebug('In test_ccpn.py: ranges: %s' % str(project.molecule.ranges)) project.molecule.rangesToMmCifRanges(ranges) if True: self.assertFalse(project.validate(htmlOnly = htmlOnly, ranges=ranges, # fastestTest=fastestTest, # disabled doProcheck = doProcheck, doWhatif = doWhatif, doWattos=doWattos, doQueeny = doQueeny, doTalos=doTalos, filterVasco=filterVasco, filterTopViolations = filterTopViolations )) if doWattos and False: mol = project.molecule completenessMol = mol.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR) nTdebug("completenessMol: %s" % completenessMol) for res in mol.allResidues(): completenessRes = res.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR) nTdebug("%s: %s" % (res, completenessRes)) # end for # end if color = project.molecule.getRogColor() nTdebug("The color of this molecule is: %s" % color) self.assertFalse( color == COLOR_RED ) # ;-) # end if # self.assertTrue(project.exportValidation2ccpn()) # self.assertFalse(project.removeCcpnReferences()) # Do not leave the old CCPN directory laying around since it might get added to by another test. # if os.path.exists(entryId): # self.assertFalse(shutil.rmtree(entryId)) if False: self.assertTrue(project.save()) self.assertTrue(project.saveCcpn(entryId)) if doRestoreCheck: del project project = Project.open(entryId, status = 'old') self.assertTrue(project, 'Failed reopening project: ' + entryId) if doStoreCheck: # # Does require below import which is used here to trigger import warning in case it's not installed. # pylint: disable=W0612 from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0404 if doStoreCING2db( entryId, ARCHIVE_NRG_ID, project=project): nTerror("Failed to store CING project's data to DB but continuing.") if doSave: self.assertTrue(project.save())
from cing.NRG import ARCHIVE_CASD_ID from cing.NRG.storeCING2db import doStoreCING2db from cing.core.classes import Project import cing import sys cing.verbosity = cing.verbosityDebug pname = sys.argv[1] project = Project.open(pname, status = 'old') project.runQueeny() project.save() doStoreCING2db( project.name, ARCHIVE_CASD_ID, project=project)
def test_queeny(self): runQueeny = 1 # DEFAULT: 1 doStoreCheck = False # DEFAULT: False Requires sqlAlchemy doValueCheck = 1 # DEFAULT: 1 Requires 1brv entryList = "1brv_cs_pk_2mdl".split() # DEFAULT because it contains many data types and is small/fast to run. # entryList = "1a24 1a4d 1afp 1ai0 1b4y 1brv 1brv 1bus 1cjg 1hue 1ieh 1iv6 1kr8 2cka 2hgh 2jzn 2k0e 8psh".split() expectedInfoList = [ (158, 0.0 ), (159, 0.0 ), (160, 0.0 ), (161, 0.0 ), (162, 0.0 ), (163, 0.0 ), (164, 0.0 ), (165, 0.000141), (166, 0.006669), (167, 0.047112), (168, 0.159379), (169, 0.309577), (170, 0.578983), (171, 0.838257), # First residue with restraints. Previously none zero elements are due to window averaging JFD assumes. (172, 1.029047), (173, 1.063531), (174, 0.969195), (175, 1.027744), (176, 0.954469), (177, 0.794625), (178, 0.822723), (179, 0.845598), (180, 0.828139), (181, 0.971881), (182, 1.141096), (183, 1.063613), (184, 0.974891), (185, 1.116405), (186, 1.113886), (187, 0.945552), (188, 0.731124), (189, 0.411986), ] cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() ) mkdirs( cingDirTmpTest ) self.failIf(os.chdir(cingDirTmpTest), msg = "Failed to change to test directory for files: " + cingDirTmpTest) for entryId in entryList: project = Project.open(entryId, status = 'new') self.assertTrue(project, 'Failed opening project: ' + entryId) inputArchiveDir = os.path.join(cingDirTestsData, "ccpn") ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz") if not os.path.exists(ccpnFile): ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz") if not os.path.exists(ccpnFile): self.fail("Neither %s or the .tgz exist" % ccpnFile) self.assertTrue(project.initCcpn(ccpnFolder = ccpnFile, modelCount=1)) if runQueeny: self.assertFalse(project.runQueeny()) if doValueCheck: for i,res in enumerate( project.molecule.allResidues()): expected = expectedInfoList[i][1] information = getDeepByKeysOrAttributes(res, QUEENY_INFORMATION_STR ) uncertainty1 = getDeepByKeysOrAttributes(res, QUEENY_UNCERTAINTY1_STR ) uncertainty2 = getDeepByKeysOrAttributes(res, QUEENY_UNCERTAINTY2_STR ) nTdebug("Comparing expected versus inf/unc1/unc2 for %20s with %8.3f %8.3f %8.3f %8.3f" % ( res, expected, information, uncertainty1, uncertainty2)) # self.assertAlmostEqual(expected,information,3) if doStoreCheck: # # Does require: try: from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0612,W0404 pdbEntryId = entryId[0:4] if doStoreCING2db( pdbEntryId, ARCHIVE_DEV_NRG_ID, project=project): nTerror("Failed to store CING project's data to DB but continuing.") except: nTwarning("Failed to doStoreCING2db from test_queeny.py")
def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = True # default: False # ranges=AUTO_STR # default is None retrieved from DBMS csv files. htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. modelCount = None # default setting is None if fastestTest: modelCount = 3 htmlOnly = True doWhatif = False doProcheck = False doWattos = False doTalos = False force_redo = True force_retrieve_input = True nTmessage(header) nTmessage(getStartMessage()) expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType', 'projectType', 'storeCING2db' ] expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTerror("Got arguments: " + repr(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % (expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True entryCodeChar2and3 = entryId[1:3] inputDir = extraArgList[0] outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[2] #@UnusedVariable restraintsConvention = extraArgList[3] archiveType = extraArgList[4] projectType = extraArgList[5] storeCING2db = False if len(extraArgList) >= expectedNumberOfArguments: storeCING2db = extraArgList[6] if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) ranges = None # targetId = getTargetForFullEntryName(entryId) # if not targetId: # nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId) # return True # ranges = getRangesForTarget(targetId) # if ranges == None: # nTerror("Failed to getRangesForTarget for targetId: %s" % targetId) # return True nTdebug("Using:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("modelCount: %s" % modelCount) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if force_redo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. # extension = '.tgz' formatFileName = '%s.tgz' # fileNameTgz = entryId + '.tgz' if projectType == PROJECT_TYPE_CING: # fileNameTgz = entryId + '.cing.tgz' formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) # if true will do retrieveTgzFromUrl. if inputDir.startswith("http") or inputDir.startswith("file"): stillToRetrieve = False if os.path.exists(fileNameTgz): if force_retrieve_input: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if # end if. # retrieveTgzFromUrl(entryId, inputDir) if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. fullFileNameTgz = os.path.join(inputDir, fileNameTgz) shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') # pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ] # for pdbFileName in pdbFileFormats: # pdbFileName = "pdb" + entryId + ".ent.gz" # # pdbFilePath = os.path.join( inputDir, pdbFileName) # pdbFilePath = os.path.join(inputDir, pdbFileName) # if os.path.exists(pdbFilePath): # break # tmpPdbFile = None # if pdbFileName.endswith('.gz'): pdbFilePath = entryId + ".pdb" # tmpPdbFile = pdbFilePath # if os.path.exists(pdbFilePath): # os.unlink(pdbFilePath) gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # if inputDirOrg == inputDirCASD_NMR: # if True: # Default is False for this is specific to CASD-NMR # nTmessage("Renaming molecule name to entry id: %s" % entryId) # project.molecule.name = entryId # insufficient since all data is already initialized to disk. # project.updateProject() # project.molecule.rename( entryId ) # project.save() # project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly. project.molecule.superpose(ranges=ranges) if True: if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doTalos=doTalos): nTerror("Failed to validate project read") return True if storeCING2db: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. try: if doStoreCING2db(entryId, ARCHIVE_CASP_ID, project=project): nTerror( "Failed to store CING project's data to DB but continuing." ) except: nTtracebackError() nTerror( "Failed to store CING project's data due to above traceback error." ) project.save() if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' # os.unlink(fileNameTgz) # temporary ccpn tgz rmdir(entryId) # temporary ccpn dir if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) do_cmd(cmd)