def _test_RetrieveTgzFromUrl(self): cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() ) mkdirs( cingDirTmpTest ) self.failIf(os.chdir(cingDirTmpTest), msg = "Failed to change to test directory for files: " + cingDirTmpTest) # url = 'http://restraintsgrid.bmrb.wisc.edu/servlet_data/NRG_ccpn_tmp' url = 'file:///Library/WebServer/Documents/NRG-CING/recoordSync' # url = 'file:/%s/ccpn' % (cingDirTestsData) entryId = '108d' # entryId = '1brv' fileNameTgz = entryId + '.tgz' if os.path.exists(fileNameTgz): os.unlink(fileNameTgz) # self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_FLAT)) self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_BY_ENTRY))
def _test_RetrieveTgzFromUrl(self): cingDirTmpTest = os.path.join(cingDirTmp, getCallerName()) mkdirs(cingDirTmpTest) self.failIf(os.chdir(cingDirTmpTest), msg="Failed to change to test directory for files: " + cingDirTmpTest) # url = 'http://restraintsgrid.bmrb.wisc.edu/servlet_data/NRG_ccpn_tmp' url = 'file:///Library/WebServer/Documents/NRG-CING/recoordSync' # url = 'file:/%s/ccpn' % (cingDirTestsData) entryId = '108d' # entryId = '1brv' fileNameTgz = entryId + '.tgz' if os.path.exists(fileNameTgz): os.unlink(fileNameTgz) # self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_FLAT)) self.assertFalse( retrieveTgzFromUrl(entryId, url, archiveType=ARCHIVE_TYPE_BY_ENTRY))
def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = True # default: False # ranges=AUTO_STR # default is None retrieved from DBMS csv files. htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. modelCount = None # default setting is None if fastestTest: modelCount = 3 htmlOnly = True doWhatif = False doProcheck = False doWattos = False doTalos = False force_redo = True force_retrieve_input = True nTmessage(cing.cingDefinitions.getHeaderString()) nTmessage(cing.systemDefinitions.getStartMessage()) expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType','projectType','storeCING2db'] expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTerror("Got arguments: " + repr(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % ( expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True entryCodeChar2and3 = entryId[1:3] inputDir = extraArgList[0] outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[2] #@UnusedVariable restraintsConvention = extraArgList[3] archiveType = extraArgList[4] projectType = extraArgList[5] storeCING2db = False if len(extraArgList) >= expectedNumberOfArguments: storeCING2db = extraArgList[6] if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) ranges = None # targetId = getTargetForFullEntryName(entryId) # if not targetId: # nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId) # return True # ranges = getRangesForTarget(targetId) # if ranges == None: # nTerror("Failed to getRangesForTarget for targetId: %s" % targetId) # return True nTdebug("Using:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("modelCount: %s" % modelCount) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if force_redo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. # extension = '.tgz' formatFileName = '%s.tgz' # fileNameTgz = entryId + '.tgz' if projectType == PROJECT_TYPE_CING: # fileNameTgz = entryId + '.cing.tgz' formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) # if true will do retrieveTgzFromUrl. if inputDir.startswith("http") or inputDir.startswith("file"): stillToRetrieve = False if os.path.exists(fileNameTgz): if force_retrieve_input: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if # end if. # retrieveTgzFromUrl(entryId, inputDir) if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. fullFileNameTgz = os.path.join(inputDir, fileNameTgz) shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') # pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ] # for pdbFileName in pdbFileFormats: # pdbFileName = "pdb" + entryId + ".ent.gz" # # pdbFilePath = os.path.join( inputDir, pdbFileName) # pdbFilePath = os.path.join(inputDir, pdbFileName) # if os.path.exists(pdbFilePath): # break # tmpPdbFile = None # if pdbFileName.endswith('.gz'): pdbFilePath = entryId + ".pdb" # tmpPdbFile = pdbFilePath # if os.path.exists(pdbFilePath): # os.unlink(pdbFilePath) gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # if inputDirOrg == inputDirCASD_NMR: # if True: # Default is False for this is specific to CASD-NMR # nTmessage("Renaming molecule name to entry id: %s" % entryId) # project.molecule.name = entryId # insufficient since all data is already initialized to disk. # project.molecule.rename( entryId ) # project.save() # project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly. project.molecule.superpose(ranges=ranges) if True: if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doTalos=doTalos): nTerror("Failed to validate project read") return True if storeCING2db: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. try: if doStoreCING2db( entryId, ARCHIVE_CASP_ID, project=project): nTerror("Failed to store CING project's data to DB but continuing.") except: nTtracebackError() nTerror("Failed to store CING project's data due to above traceback error.") project.save() if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' # os.unlink(fileNameTgz) # temporary ccpn tgz rmdir(entryId) # temporary ccpn dir if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) do_cmd(cmd)
def main(entryId, *extraArgList): """inputDir may be a directory or a url. A url needs to start with http://. """ fastestTest = True # default: False # ranges=AUTO_STR # default is None retrieved from DBMS csv files. htmlOnly = False # default: False but enable it for faster runs without some actual data. doWhatif = True # disables whatif actual run doProcheck = True doWattos = True doTalos = True tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten. modelCount = None # default setting is None if fastestTest: modelCount = 3 htmlOnly = True doWhatif = False doProcheck = False doWattos = False doTalos = False force_redo = True force_retrieve_input = True nTmessage(header) nTmessage(getStartMessage()) expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType', 'projectType', 'storeCING2db' ] expectedNumberOfArguments = len(expectedArgumentList) if len(extraArgList) != expectedNumberOfArguments: nTerror("Got arguments: " + repr(extraArgList)) nTerror("Failed to get expected number of arguments: %d got %d" % (expectedNumberOfArguments, len(extraArgList))) nTerror("Expected arguments: %s" % expectedArgumentList) return True entryCodeChar2and3 = entryId[1:3] inputDir = extraArgList[0] outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId) pdbConvention = extraArgList[2] #@UnusedVariable restraintsConvention = extraArgList[3] archiveType = extraArgList[4] projectType = extraArgList[5] storeCING2db = False if len(extraArgList) >= expectedNumberOfArguments: storeCING2db = extraArgList[6] if archiveType == ARCHIVE_TYPE_FLAT: pass # default elif archiveType == ARCHIVE_TYPE_BY_ENTRY: inputDir = os.path.join(inputDir, entryId) elif archiveType == ARCHIVE_TYPE_BY_CH23: inputDir = os.path.join(inputDir, entryCodeChar2and3) elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY: inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId) ranges = None # targetId = getTargetForFullEntryName(entryId) # if not targetId: # nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId) # return True # ranges = getRangesForTarget(targetId) # if ranges == None: # nTerror("Failed to getRangesForTarget for targetId: %s" % targetId) # return True nTdebug("Using:") nTdebug("inputDir: %s" % inputDir) nTdebug("outputDir: %s" % outputDir) nTdebug("pdbConvention: %s" % pdbConvention) nTdebug("restraintsConvention: %s" % restraintsConvention) nTdebug("archiveType: %s" % archiveType) nTdebug("projectType: %s" % projectType) nTdebug("modelCount: %s" % modelCount) nTdebug("storeCING2db: %s" % storeCING2db) nTdebug("ranges: %s" % ranges) # presume the directory still needs to be created. cingEntryDir = entryId + ".cing" if os.path.isdir(cingEntryDir): if force_redo: nTmessage("Enforcing a redo") rmtree(cingEntryDir) else: mainIndexFile = os.path.join(cingEntryDir, "index.html") isDone = os.path.isfile(mainIndexFile) if isDone: nTmessage("SKIPPING ENTRY ALREADY DONE") return nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.") rmtree(cingEntryDir) # end if. # end if. os.chdir(outputDir) project = Project(entryId) if project.removeFromDisk(): nTerror("Failed to remove existing project (if present)") return True # end if. # extension = '.tgz' formatFileName = '%s.tgz' # fileNameTgz = entryId + '.tgz' if projectType == PROJECT_TYPE_CING: # fileNameTgz = entryId + '.cing.tgz' formatFileName = '%s.cing.tgz' elif projectType == PROJECT_TYPE_PDB: formatFileName = 'pdb%s.ent.gz' fileNameTgz = formatFileName % entryId # nTdebug("fileNameTgz: %s" % fileNameTgz) # if true will do retrieveTgzFromUrl. if inputDir.startswith("http") or inputDir.startswith("file"): stillToRetrieve = False if os.path.exists(fileNameTgz): if force_retrieve_input: os.unlink(fileNameTgz) stillToRetrieve = True # end if else: stillToRetrieve = True # end if if stillToRetrieve: retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName) # end if if not os.path.exists(fileNameTgz): nTerror("Tgz should already have been present skipping entry") return # end if # end if. # retrieveTgzFromUrl(entryId, inputDir) if projectType == PROJECT_TYPE_CING: # Needs to be copied because the open method doesn't take a directory argument.. fullFileNameTgz = os.path.join(inputDir, fileNameTgz) shutil.copy(fullFileNameTgz, '.') project = Project.open(entryId, status='old') if not project: nTerror("Failed to init old project") return True elif projectType == PROJECT_TYPE_CCPN: project = Project.open(entryId, status='new') if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount): nTerror("Failed to init project from ccpn") return True elif projectType == PROJECT_TYPE_PDB: project = Project.open(entryId, status='new') # pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ] # for pdbFileName in pdbFileFormats: # pdbFileName = "pdb" + entryId + ".ent.gz" # # pdbFilePath = os.path.join( inputDir, pdbFileName) # pdbFilePath = os.path.join(inputDir, pdbFileName) # if os.path.exists(pdbFilePath): # break # tmpPdbFile = None # if pdbFileName.endswith('.gz'): pdbFilePath = entryId + ".pdb" # tmpPdbFile = pdbFilePath # if os.path.exists(pdbFilePath): # os.unlink(pdbFilePath) gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True) project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount) # if tmpPdbFile: if True: nTdebug("Removing tmp: %s" % pdbFilePath) os.unlink(pdbFilePath) # if inputDirOrg == inputDirCASD_NMR: # if True: # Default is False for this is specific to CASD-NMR # nTmessage("Renaming molecule name to entry id: %s" % entryId) # project.molecule.name = entryId # insufficient since all data is already initialized to disk. # project.updateProject() # project.molecule.rename( entryId ) # project.save() # project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly. project.molecule.superpose(ranges=ranges) if True: if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif, doWattos=doWattos, doTalos=doTalos): nTerror("Failed to validate project read") return True if storeCING2db: # Does require: #from cing.PluginCode.sqlAlchemy import csqlAlchemy # and should never crash run. try: if doStoreCING2db(entryId, ARCHIVE_CASP_ID, project=project): nTerror( "Failed to store CING project's data to DB but continuing." ) except: nTtracebackError() nTerror( "Failed to store CING project's data due to above traceback error." ) project.save() if projectType == PROJECT_TYPE_CCPN: # fileNameTgz = entryId + '.tgz' # os.unlink(fileNameTgz) # temporary ccpn tgz rmdir(entryId) # temporary ccpn dir if tgzCing: directoryNameCing = entryId + ".cing" tgzFileNameCing = directoryNameCing + ".tgz" if os.path.exists(tgzFileNameCing): nTwarning("Overwriting: " + tgzFileNameCing) cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing) do_cmd(cmd)