def testCopyHTMLFilesAndDir(self): if not os.path.exists(EDUtilsPath.getEdnaUserTempFolder()): os.mkdir(EDUtilsPath.getEdnaUserTempFolder()) strTestFromDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestFromDir") shutil.rmtree(strTestFromDir, ignore_errors=True) os.mkdir(strTestFromDir) strTestHtmlFilePath = os.path.join(strTestFromDir, "index.html") strTestHtmlDirPath = os.path.join(strTestFromDir, "index") EDUtilsFile.writeFile(strTestHtmlFilePath, "Test content") if not os.path.exists(strTestHtmlDirPath): os.mkdir(strTestHtmlDirPath) strTestHtmlDirFilePath = os.path.join(strTestHtmlDirPath, "test.txt") EDUtilsFile.writeFile(strTestHtmlDirFilePath, "Test content") # strTestToDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestToDir") shutil.rmtree(strTestToDir, ignore_errors=True) os.mkdir(strTestToDir) EDHandlerESRFPyarchv1_0.copyHTMLDir(strTestFromDir, strTestToDir) # # Check that files exist in strTestToDir: EDAssert.isFile(os.path.join(strTestToDir, "index", "index.html")) EDAssert.isFile( os.path.join(strTestToDir, "index", "index", "test.txt")) # shutil.rmtree(strTestFromDir, ignore_errors=True) shutil.rmtree(strTestToDir, ignore_errors=True)
def testCopyHTMLFilesAndDir(self): if not os.path.exists(EDUtilsPath.getEdnaUserTempFolder()): os.mkdir(EDUtilsPath.getEdnaUserTempFolder()) strTestFromDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestFromDir") shutil.rmtree(strTestFromDir, ignore_errors=True) os.mkdir(strTestFromDir) strTestHtmlFilePath = os.path.join(strTestFromDir, "index.html") strTestHtmlDirPath = os.path.join(strTestFromDir, "index") EDUtilsFile.writeFile(strTestHtmlFilePath, "Test content") if not os.path.exists(strTestHtmlDirPath): os.mkdir(strTestHtmlDirPath) strTestHtmlDirFilePath = os.path.join(strTestHtmlDirPath, "test.txt") EDUtilsFile.writeFile(strTestHtmlDirFilePath, "Test content") # strTestToDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestToDir") shutil.rmtree(strTestToDir, ignore_errors=True) os.mkdir(strTestToDir) EDHandlerESRFPyarchv1_0.copyHTMLDir(strTestFromDir, strTestToDir) # # Check that files exist in strTestToDir: EDAssert.isFile(os.path.join(strTestToDir, "index", "index.html")) EDAssert.isFile(os.path.join(strTestToDir, "index", "index", "test.txt")) # shutil.rmtree(strTestFromDir, ignore_errors=True) shutil.rmtree(strTestToDir, ignore_errors=True)
def finallyProcess(self, _edPlugin=None): EDPluginExec.finallyProcess(self, _edPlugin) self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...") xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage() xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName)))) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory()))) # Write workflowStepReport HTML page pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName) pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory()) # Store in Pyarch if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV() or EDUtilsPath.isALBA: strPyarchPath = None if self.xsDataResultCharacterisation is not None: strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection()) if strPyarchPath is None: # For debugging purposes strPyarchPath = EDUtilsPath.getEdnaUserTempFolder() EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath))) if not os.path.exists(strPyarchPath): os.makedirs(strPyarchPath, 0o755) shutil.copy(pathToJsonFile, strPyarchPath) pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile)) # Write json file xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile)) self.setDataOutput(xsDataResultSimpleHTMLPage)
def finallyProcess(self, _edPlugin=None): EDPluginExec.finallyProcess(self, _edPlugin) self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...") xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage() xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName)))) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory()))) # Write workflowStepReport HTML page pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName) pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory()) # Store in Pyarch if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL(): strPyarchPath = None if self.xsDataResultCharacterisation is not None: strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection()) if strPyarchPath is None: # For debugging purposes strPyarchPath = EDUtilsPath.getEdnaUserTempFolder() EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath))) if not os.path.exists(strPyarchPath): os.makedirs(strPyarchPath, 0o755) shutil.copy(pathToJsonFile, strPyarchPath) pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile)) # Write json file xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile)) self.setDataOutput(xsDataResultSimpleHTMLPage)
def finallyProcess(self, _edPlugin=None): EDPluginExec.finallyProcess(self, _edPlugin) self.DEBUG("EDPluginExecSimpleHTMLPagev2_0.finallyProcess...") # Render the page strHTML = str(self.page) EDUtilsFile.writeFile(self.strPath, strHTML) xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage() xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(self.strPath))) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(os.path.dirname(self.strPath)))) self.setDataOutput(xsDataResultSimpleHTMLPage) # Store in Pyarch strPyarchPath = None if self.xsDataResultCharacterisation is not None: strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection()) if strPyarchPath is None: # For debugging purposes strPyarchPath = EDUtilsPath.getEdnaUserTempFolder() EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
def testCreatePyarchHtmlDirectoryPath(self): strTestDataDir = self.getPluginTestsDataHome() strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml") strXml = self.readAndParseFile(strTestFile) xsDataCollection = XSDataCollection.parseString(strXml) strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(xsDataCollection) # print strPyarchHtmlDirectoryPath strReferencePath = "/data/pyarch/2016/id30b/opid30b/20161214/RAW_DATA/Magda/Test_x1_2_dnafiles" EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def finallyProcess(self, _edPlugin=None): EDPluginExec.finallyProcess(self, _edPlugin) self.DEBUG("EDPluginExecSimpleHTMLPagev1_0.finallyProcess...") # Render the page strHTML = str(self.page) EDUtilsFile.writeFile(self.strPath, strHTML) xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage() xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(self.strPath))) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(os.path.dirname(self.strPath)))) self.setDataOutput(xsDataResultSimpleHTMLPage) # Store in Pyarch strPyarchPath = None if self.xsDataResultCharacterisation is not None: strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection()) if strPyarchPath is None: # For debugging purposes strPyarchPath = EDUtilsPath.getEdnaUserTempFolder() EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
def testCreatePyarchHtmlDirectoryPath(self): strTestDataDir = self.getPluginTestsDataHome() strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml") strXml = self.readAndParseFile(strTestFile) xsDataCollection = XSDataCollection.parseString(strXml) strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(xsDataCollection) #print strPyarchHtmlDirectoryPath strReferencePath = "/data/pyarch/id23eh1/Pasha/data/visitor/mx1199/id29/20111205/RAW_DATA/4ESR372D11/4ESR372D11_1_dnafiles" EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def testCreatePyarchHtmlDirectoryPath(self): strTestDataDir = self.getPluginTestsDataHome() strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml") strXml = self.readAndParseFile(strTestFile) xsDataCollection = XSDataCollection.parseString(strXml) strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath( xsDataCollection) # print strPyarchHtmlDirectoryPath strReferencePath = "/data/pyarch/2016/id30b/opid30b/20161214/RAW_DATA/Magda/Test_x1_2_dnafiles" EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def testCreatePyarchReprocessDirectoryPath(self): beamline = "id30a2" pipelineName = "EDNA_proc" dataCollectionId = 123456 pyarch_path = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath( beamline, pipelineName, dataCollectionId) EDAssert.equal(True, os.path.exists(pyarch_path)) EDAssert.equal( True, pyarch_path.startswith( "/data/pyarch/2020/id30a2/reprocess/EDNA_proc/123456")) shutil.rmtree(pyarch_path)
def testCreatePyarchReprocessDirectoryPath(self): beamline = "id30a2" pipelineName = "EDNA_proc" dataCollectionId = 123456 pyarch_path = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath( beamline, pipelineName, dataCollectionId) EDAssert.equal(True, os.path.exists(pyarch_path)) year = time.strftime("%Y", time.localtime(time.time())) EDAssert.equal( True, pyarch_path.startswith( "/data/pyarch/{0}/id30a2/reprocess/EDNA_proc/123456".format( year))) shutil.rmtree(pyarch_path)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0o755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]: setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # Attached files pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory) pyarchResultPath = os.path.join(pyarchPath, "results") if not os.path.exists(pyarchResultPath): os.makedirs(pyarchResultPath, 0o755) # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0755) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() # Prepare input to execution plugin xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS() xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS() xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath))) else: xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage))) timeStart = time.localtime() self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom self.edPluginExecXia2DIALSAnom.execute() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom self.edPluginExecXia2DIALSNoanom.execute() self.edPluginExecXia2DIALSAnom.synchronize() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd) if self.doAnomAndNonanom: self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
def testCreatePyarchFilePath(self): EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2")) # Test with inhouse account... EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2")) EDAssert.equal("/data/pyarch/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles")) # Test with gzvisitor EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gzvisitor/mx415/id30a3")) EDAssert.equal("/data/pyarch/id30a3/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212")) EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1")) EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1/2")) EDAssert.equal("/data/pyarch/id30a3/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525")) EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1")) EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1/2")) EDAssert.equal("/data/pyarch/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"))
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace( 'PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists( self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0o755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[ scalingStatisticsType]: setattr( autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType] [scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics( autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # Attached files pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath( processDirectory) pyarchResultPath = os.path.join(pyarchPath, "results") if not os.path.exists(pyarchResultPath): os.makedirs(pyarchResultPath, 0o755) # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename( mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext( os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.dataInput.diffractionImage.path.value strImageFileNameExtension = os.path.splitext( strPathToDiffractionImage)[1] if not strImageFileNameExtension in [ ".img", ".marccd", ".mccd", ".cbf", ".h5" ]: self.error( "Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the MXWaitFile plugin xsDataInputMXWaitFile = XSDataInputMXWaitFile() pathToImageFile = strPathToDiffractionImage # Quite ugly hack to avoid lag problems at the ESRF: if EDUtilsPath.isESRF() or EDUtilsPath.isALBA(): if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]): # Pilatus 6M self.minImageSize = 6000000 elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]): # Pilatus3 2M self.minImageSize = 2000000 elif strImageFileNameExtension == ".h5": self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath( pathToImageFile) pathToImageFile = self.h5DataFilePath self.isH5 = True elif EDUtilsPath.isEMBL(): self.minImageSize = 10000 xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setFile( XSDataFile(XSDataString(pathToImageFile))) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputMXWaitFile.setTimeOut( self.getDataInput().getWaitForFileTimeOut()) self.edPluginMXWaitFile = self.loadPlugin( self.strMXWaitFilePluginName) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) # Load the execution plugin self.edPluginExecThumbnail = self.loadPlugin( self.strExecThumbnailPluginName) xsDataInputMXThumbnail = XSDataInputMXThumbnail() xsDataInputMXThumbnail.image = self.getDataInput( ).getDiffractionImage() xsDataInputMXThumbnail.height = XSDataInteger(1024) xsDataInputMXThumbnail.width = XSDataInteger(1024) xsDataInputMXThumbnail.format = self.dataInput.format # Output path strImageNameWithoutExt = os.path.basename( os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput( ).getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join( strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath( strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except Exception as e: self.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: self.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0o755) strOutputDirname = tempfile.mkdtemp( prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser) os.chmod(strOutputDirname, 0o755) self.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join( strOutputDirname, strImageNameWithoutExt) if self.dataInput.format is not None: self.strSuffix = self.dataInput.format.value.lower() self.strImageFormat = self.dataInput.format.value.upper() self.strOutputPath = os.path.join( self.strOutputPathWithoutExtension + "." + self.strSuffix) xsDataInputMXThumbnail.setOutputPath( XSDataFile(XSDataString(self.strOutputPath))) self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def testCreatePyarchFilePath(self): EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"), "/") EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"), "/data") EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"), "/data/visitor") EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id14eh2"), "/data/visitor/mx415/id14eh2") EDAssert.equal( "/data/pyarch/2010/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id14eh2/20100212"), "/data/visitor/mx415/id14eh2/20100212") EDAssert.equal( "/data/pyarch/2010/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id14eh2/20100212/1"), "/data/visitor/mx415/id14eh2/20100212/1") EDAssert.equal( "/data/pyarch/2010/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id14eh2/20100212/1/2"), "/data/visitor/mx415/id14eh2/20100212/1/2") # Test with inhouse account... EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"), "/") EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"), "/data") EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"), "/data/id23eh2") EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id23eh2/inhouse"), "/data/id23eh2/inhouse") EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id23eh2/inhouse/opid232"), "/data/id23eh2/inhouse/opid232") EDAssert.equal( "/data/pyarch/2010/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id23eh2/inhouse/opid232/20100525"), "/data/id23eh2/inhouse/opid232/20100525") EDAssert.equal( "/data/pyarch/2010/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id23eh2/inhouse/opid232/20100525/1"), "/data/id23eh2/inhouse/opid232/20100525/1") EDAssert.equal( "/data/pyarch/2010/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id23eh2/inhouse/opid232/20100525/1/2"), "/data/id23eh2/inhouse/opid232/20100525/1/2") EDAssert.equal( "/data/pyarch/2014/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles" ), "/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles" ) # Visitor EDAssert.equal( None, EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id30a3"), "/data/visitor/mx415/id30a3") EDAssert.equal( "/data/pyarch/2010/id30a3/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id30a3/20100212"), "/data/visitor/mx415/id30a3/20100212") EDAssert.equal( "/data/pyarch/2010/id30a3/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id30a3/20100212/1"), "/data/visitor/mx415/id30a3/20100212/1") EDAssert.equal( "/data/pyarch/2010/id30a3/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/visitor/mx415/id30a3/20100212/1/2"), "/data/visitor/mx415/id30a3/20100212/1/2") EDAssert.equal( "/data/pyarch/2010/id30a3/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id30a3/inhouse/opid232/20100525"), "/data/id30a3/inhouse/opid232/20100525") EDAssert.equal( "/data/pyarch/2010/id30a3/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id30a3/inhouse/opid232/20100525/1"), "/data/id30a3/inhouse/opid232/20100525/1") EDAssert.equal( "/data/pyarch/2010/id30a3/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id30a3/inhouse/opid232/20100525/1/2"), "/data/id30a3/inhouse/opid232/20100525/1/2") EDAssert.equal( "/data/pyarch/2014/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath( "/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles" ), "/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles" )
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) EDVerbose.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.getDataInput().getDiffractionImage().getPath().getValue() strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1] if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf"]: print strImageFileNameExtension EDVerbose.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the waitFile plugin xsDataInputWaitFile = XSDataInputWaitFile() xsDataInputWaitFile.setExpectedSize(XSDataInteger(self.__iExpectedSize)) xsDataInputWaitFile.setExpectedFile(self.getDataInput().getDiffractionImage()) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut()) self.__edPluginWaitFile = EDPluginWaitFile() self.__edPluginWaitFile.setDataInput(xsDataInputWaitFile) # Load the execution plugin self.__edPluginExecThumbnail = self.loadPlugin(self.__strExecThumbnailPluginName) xsDataInputExecThumbnail = XSDataInputExecThumbnail() xsDataInputExecThumbnail.setInputImagePath(self.getDataInput().getDiffractionImage()) xsDataInputExecThumbnail.setLevelsInvert(XSDataBoolean(True)) xsDataInputExecThumbnail.setLevelsMin(XSDataDoubleWithUnit(0.0)) xsDataDoubleWithUnitLevelsMax = XSDataDoubleWithUnit(99.95) xsDataDoubleWithUnitLevelsMax.setUnit(XSDataString("%")) xsDataInputExecThumbnail.setLevelsMax(xsDataDoubleWithUnitLevelsMax) xsDataInputExecThumbnail.setFilterDilatation([XSDataInteger(4)]) xsDataInputExecThumbnail.setLevelsColorize(XSDataBoolean(False)) xsDataInputExecThumbnail.setThumbHeight(XSDataInteger(1024)) xsDataInputExecThumbnail.setThumbWidth(XSDataInteger(1024)) xsDataInputExecThumbnail.setKeepRatio(XSDataBoolean(False)) # Output path strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): EDVerbose.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except BaseException, e: EDVerbose.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: EDVerbose.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strOutputDirname = tempfile.mkdtemp("", "EDPluginPyarchThumbnailv10_", "/tmp") EDVerbose.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt) self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + ".jpeg") xsDataInputExecThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath))) self.__edPluginExecThumbnail.setDataInput(xsDataInputExecThumbnail)
def storeResultsInISPyB(self, _strSubject, _strMessage): strSubject = _strSubject strMessage = _strMessage xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation() self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation) xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB() if xsDataResultControlISPyB != None: self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId()) if xsDataResultCharacterisation != None: self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml") xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult) self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult)) # For the moment, create "DNA" style output directory strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation) xsDataDictionaryLogFile = None if (self.createDNAFileDirectory(strPathToDNAFileDirectory)): xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPathToDNAFileDirectory) strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory) if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)): xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory) self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile) if xsDataResultCharacterisation.getStatusMessage(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getStatusMessage().getValue() if xsDataResultCharacterisation.getShortSummary(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getShortSummary().getValue() self.sendEmail(strSubject, strMessage) # Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run bRunExecOutputHTML = False xsDataInputMXCuBE = self.getDataInput() xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan() if xsDataDiffractionPlan.getStrategyOption() is not None: strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue() if strStrategyOption.find("-DamPar") != -1: bRunExecOutputHTML = False if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML: self.edPluginExecOutputHTML.setDataInput(XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory") self.edPluginExecOutputHTML.execute() # Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult() if xsDataIntegrationResult: listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult() for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult: if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None: strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue() # Put the file one directory above the mxCuBE v1.3 plugin working directory: strDir = os.path.dirname(self.getWorkingDirectory()) strPath = os.path.join(strDir, "bestfile.par") EDUtilsFile.writeFile(strPath, strBestfilePar) break # Execute plugin which creates a simple HTML page self.executeSimpleHTML(xsDataResultCharacterisation) # Upload the best wilson plot path to ISPyB strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation) if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None: # Copy wilson path to Pyarch strBestWilsonPlotPyarchPath = os.path.join(strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)) if not os.path.exists(strBestWilsonPlotPyarchPath): if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)): os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755) shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath) self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath) if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None: xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath() xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath) edPluginSetBestWilsonPlotPath = self.loadPlugin("EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath") edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath edPluginSetBestWilsonPlotPath.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlAutoPROCv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5) elif EDUtilsPath.isMAXIV(): template = ispybDataCollection.fileTemplate else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") if self.dataInput.fromN is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.fromN.value if self.dataInput.toN is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.toN.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.path.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value if EDUtilsPath.isEMBL(): fileTemplate = template.replace("#####", "%05d") else: fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory if EDUtilsPath.isALBA(): _processDirectory = "_".join(pathToStartImage.split('_')[:-1]) from datetime import datetime _id = datetime.now().strftime('%Y%m%d_%H%M%S') self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id) else: self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "autoPROC", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # The resultsDirectory is not used at ALBA (only pyarchDirectory) if EDUtilsPath.isALBA(): self.resultsDirectory = None # Determine pyarch prefix if EDUtilsPath.isALBA(): listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]), listPrefix[-2]) else: listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isMAXIV(): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV(): fWaitFileTimeout = 60 # s else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=True, timeStart=self.timeStart) self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=False, timeStart=self.timeStart) self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=False, timeStart=self.timeStart) # Prepare input to execution plugin if self.doAnom: xsDataInputAutoPROCAnom = XSDataInputAutoPROC() xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True) xsDataInputAutoPROCAnom.symm = self.dataInput.symm xsDataInputAutoPROCAnom.cell = self.dataInput.cell xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit if self.doNoanom: xsDataInputAutoPROCNoanom = XSDataInputAutoPROC() xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False) xsDataInputAutoPROCNoanom.symm = self.dataInput.symm xsDataInputAutoPROCNoanom.cell = self.dataInput.cell xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier() xsDataAutoPROCIdentifier.idN = XSDataString(identifier) xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory)) xsDataAutoPROCIdentifier.templateN = XSDataString(template) xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart) xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd) if self.doAnom: xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier) if self.doNoanom: xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy()) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath)) if self.doNoanom: xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath)) timeStart = time.localtime() if self.doAnom: self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom self.edPluginExecAutoPROCAnom.execute() if self.doNoanom: self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom self.edPluginExecAutoPROCNoanom.execute() if self.doAnom: self.edPluginExecAutoPROCAnom.synchronize() if self.doNoanom: self.edPluginExecAutoPROCNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd) if self.doNoanom: self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXDSAPPv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" if self.dataInput.startImageNumber is not None: imageNoStart = self.dataInput.startImageNumber.value if self.dataInput.endImageNumber is not None: imageNoEnd = self.dataInput.endImageNumber.value # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") if imageNoStart is None: imageNoStart = ispybDataCollection.startImageNumber if imageNoEnd is None: imageNoEnd = ispybDataCollection.startImageNumber + \ ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "XDSAPP", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() if self.dataInput.hdf5ToCbfDirectory is not None: dir = self.dataInput.hdf5ToCbfDirectory.path.value pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0] self.timeStart = time.localtime() # Prepare input to execution plugin if self.doAnom: xsDataInputXDSAPPAnom = XSDataInputXDSAPP() xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True) xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup) self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom self.edPluginExecXDSAPPAnom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doNoanom: xsDataInputXDSAPPNoanom = XSDataInputXDSAPP() xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False) xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup) self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom self.edPluginExecXDSAPPNoanom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) if self.doAnom: self.edPluginExecXDSAPPAnom.synchronize() xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput # Run XSCALE even if XSCALE.LP is present strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True) if self.doNoanom: self.edPluginExecXDSAPPNoanom.synchronize() xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True) self.timeEnd = time.localtime() # Upload to ISPyB if self.dataInput.dataCollectionId is not None: # Check if we should use XDS_ASCII_to_XML.pl if self.doAnom: if self.useXdsAsciiToXml: # Only for anom runs self.runXdsAsciiToXml(xsDataResultXDSAPPAnom, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) else: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template, strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doNoanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template, strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlCrystFELv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None userName = os.environ["USER"] if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB #self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value, # "CrystFEL", # "started") xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection( ) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection self.ispyb_retrieve_dc_plugin.executeSynchronous() ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): #TODO PE2 has 7 digits others 5 template = ispybDataCollection.fileTemplate.replace( "%07d", "####") else: template = ispybDataCollection.fileTemplate.replace( "%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") dir_parts = processDirectory.split("/") processDirectory = "/data/users/%s" % os.path.join(*dir_parts[5:]) # Make results directory self.resultsDirectory = os.path.join(processDirectory, "crystfel_results") try: if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory) except Exception as ex: print ex self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( directory) #TODO: enable when ready """ if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory) """ self.baseFileName = "%s/crystfel_xgandalf_%d" % ( self.resultsDirectory, self.dataInput.dataCollectionId.value) xsDataInputCrystFEL = XSDataInputCrystFEL() # Check if imagesFullPath exists. If not then generate one if self.dataInput.imagesFullPath is None: imagesFullPath = "%s_images_fullpath.lst" % self.baseName imagesFullFile = open(imagesFullPath, "w") for index in range(imageNoEnd - imageNoStart + 1): imagePath = os.path.join( directory, ispybDataCollection.fileTemplate % (index + 1)) imagesFullFile.write("%s\n" % imagePath) imagesFullFile.close() xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath) else: xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath xsDataInputCrystFEL.geomFile = self.dataInput.geomFile xsDataInputCrystFEL.cellFile = self.dataInput.cellFile xsDataInputCrystFEL.pointGroup = self.dataInput.pointGroup xsDataInputCrystFEL.spaceGroup = self.dataInput.spaceGroup xsDataInputCrystFEL.resCutOff = self.dataInput.resCutOff xsDataInputCrystFEL.baseFileName = XSDataString(self.baseFileName) self.timeStart = time.localtime() self.integration_id, self.program_id = self.create_integration_id( self.dataInput.dataCollectionId.value, "Creating integration ID") self.index_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL self.partialator_plugin.dataInput = xsDataInputCrystFEL self.post_process_plugin.dataInput = xsDataInputCrystFEL self.post_process_plugin.dataOutput = XSDataResultCrystFEL self.process_hkl_plugin_odd.process_hkl_options += " --odd-only" self.process_hkl_plugin_odd.process_hkl_type = "o" self.process_hkl_plugin_even.process_hkl_options += " --even-only" self.process_hkl_plugin_even.process_hkl_type = "e" self.index_plugin.executeSynchronous() if self.index_plugin.isFailure(): self.ERROR('indexamajig: Failed') self.setFailure() return else: self.screen('indexamajig: Finished') self.process_hkl_plugin.executeSynchronous() if self.process_hkl_plugin.isFailure(): self.ERROR('process_hkl: Failed') self.setFailure() return else: self.screen('process_hkl: Finished') self.process_hkl_plugin_odd.executeSynchronous() if self.process_hkl_plugin_odd.isFailure(): self.ERROR('process_hkl_odd: Failed') self.setFailure() return else: self.screen('process_hkl_odd: Finished') self.process_hkl_plugin_even.executeSynchronous() if self.process_hkl_plugin_even.isFailure(): self.ERROR('process_hkl_even: Failed') self.setFailure() return else: self.screen('process_hkl_even: Finished') self.partialator_plugin.executeSynchronous() if self.partialator_plugin.isFailure(): self.ERROR('partialator: Failed') self.setFailure() return else: self.screen('partialator: Finished') self.post_process_plugin.executeSynchronous() if self.post_process_plugin.isFailure(): self.ERROR('post_process: Failed') self.setFailure() return else: self.screen('post_process: Finished') self.timeEnd = time.localtime()
def storeResultsInISPyB(self, _strSubject, _strMessage): strSubject = _strSubject strMessage = _strMessage xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput( ).getResultCharacterisation() self.xsDataResultMXCuBE.setCharacterisationResult( xsDataResultCharacterisation) xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput( ).getResultControlISPyB() if xsDataResultControlISPyB != None: self.xsDataResultMXCuBE.setScreeningId( xsDataResultControlISPyB.getScreeningId()) if xsDataResultCharacterisation != None: self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation strPathCharacterisationResult = os.path.join( self.getWorkingDirectory(), "CharacterisationResult.xml") xsDataResultCharacterisation.exportToFile( strPathCharacterisationResult) self.xsDataResultMXCuBE.setListOfOutputFiles( XSDataString(strPathCharacterisationResult)) # For the moment, create "DNA" style output directory strPathToDNAFileDirectory = self.createDNAFileDirectoryPath( xsDataResultCharacterisation) xsDataDictionaryLogFile = None if not EDUtilsPath.isALBA: if (self.createDNAFileDirectory(strPathToDNAFileDirectory)): xsDataDictionaryLogFile = self.createOutputFileDictionary( xsDataResultCharacterisation, strPathToDNAFileDirectory) strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( strPathToDNAFileDirectory) if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)): xsDataDictionaryLogFile = self.createOutputFileDictionary( xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory) self.xsDataResultMXCuBE.setOutputFileDictionary( xsDataDictionaryLogFile) if xsDataResultCharacterisation.getStatusMessage(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getStatusMessage( ).getValue() if xsDataResultCharacterisation.getShortSummary(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getShortSummary( ).getValue() self.sendEmail(strSubject, strMessage) # Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run bRunExecOutputHTML = False xsDataInputMXCuBE = self.getDataInput() xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan() if xsDataDiffractionPlan.getStrategyOption() is not None: strStrategyOption = xsDataDiffractionPlan.getStrategyOption( ).getValue() if strStrategyOption.find("-DamPar") != -1: bRunExecOutputHTML = False if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML: self.edPluginExecOutputHTML.setDataInput( XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory") self.edPluginExecOutputHTML.execute() # Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult( ) if xsDataIntegrationResult: listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult( ) for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult: if xsDataIntegrationSubWedgeResult.getBestfilePar( ) is not None: strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar( ).getValue() # Put the file one directory above the mxCuBE v1.3 plugin working directory: strDir = os.path.dirname(self.getWorkingDirectory()) strPath = os.path.join(strDir, "bestfile.par") EDUtilsFile.writeFile(strPath, strBestfilePar) break # Execute plugin which creates a simple HTML page self.executeSimpleHTML(xsDataResultCharacterisation) # Upload the best wilson plot path to ISPyB strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath( xsDataResultCharacterisation) if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None: # Copy wilson path to Pyarch strBestWilsonPlotPyarchPath = os.path.join( strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)) if not os.path.exists(strBestWilsonPlotPyarchPath): if not os.path.exists( os.path.dirname(strBestWilsonPlotPyarchPath)): os.makedirs( os.path.dirname(strBestWilsonPlotPyarchPath), 755) shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath) self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath) if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None: xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath( ) xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString( strBestWilsonPlotPyarchPath) edPluginSetBestWilsonPlotPath = self.loadPlugin( "EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath") edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath edPluginSetBestWilsonPlotPath.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlCrystFELv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB #self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value, # "CrystFEL", # "started") xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection self.ispyb_retrieve_dc_plugin.executeSynchronous() ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): #TODO PE2 has 7 digits others 5 template = ispybDataCollection.fileTemplate.replace("%07d", "####") else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") try: if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 755) except Exception as ex: print ex # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) #TODO: enable when ready """ if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 755) """ # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 60 xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst) self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst self.wait_first_image_plugin.executeSynchronous() if self.wait_first_image_plugin.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory self.streamFilename = "%s_%d.stream" % ( self.baseName, self.dataInput.dataCollectionId.value ) self.hklFilename = "%s_%d.hkl" % ( self.baseName, self.dataInput.dataCollectionId.value ) self.mtzFilename = "%s_%d.mtz" % ( self.baseName, self.dataInput.dataCollectionId.value ) # Prepare input to execution plugin xsDataInputCrystFEL = XSDataInputCrystFEL() # Check if imagesFullPath exists. If not then generate one if self.dataInput.imagesFullPath is None: imagesFullPath = "%s_images_fullpath.lst" % self.baseName imagesFullFile = open(imagesFullPath, "w") for index in range(imageNoEnd - imageNoStart + 1): imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1)) imagesFullFile.write("%s\n" % imagePath) imagesFullFile.close() xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath) else: xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath xsDataInputCrystFEL.geomFile = self.dataInput.geomFile xsDataInputCrystFEL.cellFile = self.dataInput.cellFile #TODO EDNA can generate the imagesFull path if id do not exist xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename) xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename) xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename) self.timeStart = time.localtime() """ if self.dataInput.dataCollectionId is not None: # Set ISPyB to running self.autoProcIntegrationId, self.autoProcProgramId = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) """ self.index_plugin.dataInput = xsDataInputCrystFEL self.index_plugin.executeSynchronous() if self.index_plugin.isFailure(): self.ERROR('indexamajig: Failed') self.setFailure() return else: self.screen('indexamajig: Finished') self.process_hkl_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL self.partialator_plugin.dataInput = xsDataInputCrystFEL self.post_process_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_odd.process_hkl_options += " --odd-only" self.process_hkl_plugin_even.process_hkl_options += " --even-only" self.process_hkl_plugin.executeSynchronous() if self.process_hkl_plugin.isFailure(): self.ERROR('process_hkl: Failed') self.setFailure() return else: self.screen('process_hkl: Finished') self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o") self.process_hkl_plugin_odd.executeSynchronous() if self.process_hkl_plugin_odd.isFailure(): self.ERROR('process_hkl_odd: Failed') self.setFailure() return else: self.screen('process_hkl_odd: Finished') self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e") self.process_hkl_plugin_even.executeSynchronous() if self.process_hkl_plugin_even.isFailure(): self.ERROR('process_hkl_even: Failed') self.setFailure() return else: self.screen('process_hkl_even: Finished') self.partialator_plugin.executeSynchronous() if self.partialator_plugin.isFailure(): self.ERROR('partialator: Failed') self.setFailure() return else: self.screen('partialator: Finished') self.post_process_plugin.executeSynchronous() if self.post_process_plugin.isFailure(): self.ERROR('post_process: Failed') self.setFailure() return else: self.screen('post_process: Finished') """ self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL self.edPluginExecCrystFELScale.executeSynchronous() if self.edPluginExecCrystFELScale.isFailure(): self.ERROR('CrystFELScalev1_0: Failed') self.setFailure() return else: self.screen('CrystFELScalev1_0: Finished') self.timeEnd = time.localtime() """ # Upload to ISPyB """
def testCreatePyarchFilePath(self): EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor")) EDAssert.equal("/data/pyarch/id14eh2/mx415", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1")) EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2")) # Test with inhouse account... EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2")) EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse")) EDAssert.equal("/data/pyarch/id23eh2/opid232", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1")) EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2"))
def storeResultsInISPyB(self, _strSubject, _strMessage): strSubject = _strSubject strMessage = _strMessage xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation() if self.bIsEigerDetector: xsDataResultCharacterisation = self.makeNumberOfImagesMultipleOf100(xsDataResultCharacterisation) self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation) xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB() if xsDataResultControlISPyB != None: self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId()) if xsDataResultCharacterisation != None: self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml") xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult) self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult)) # For the moment, create "DNA" style output directory strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation) xsDataDictionaryLogFile = None if self.createDNAFileDirectory(strPathToDNAFileDirectory): xsDataDictionaryLogFile = self.createOutputFileDictionary( xsDataResultCharacterisation, strPathToDNAFileDirectory ) strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory) if self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory): xsDataDictionaryLogFile = self.createOutputFileDictionary( xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory ) self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile) if xsDataResultCharacterisation.getStatusMessage(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getStatusMessage().getValue() if xsDataResultCharacterisation.getShortSummary(): strMessage += "\n\n" strMessage += xsDataResultCharacterisation.getShortSummary().getValue() self.sendEmail(strSubject, strMessage) # Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run bRunExecOutputHTML = False xsDataInputMXCuBE = self.getDataInput() xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan() if xsDataDiffractionPlan is not None and xsDataDiffractionPlan.getStrategyOption() is not None: strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue() if strStrategyOption.find("-DamPar") != -1: bRunExecOutputHTML = False if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML: self.edPluginExecOutputHTML.setDataInput( XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory" ) self.edPluginExecOutputHTML.execute() # Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult() if xsDataIntegrationResult: listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult() for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult: if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None: strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue() # Put the file one directory above the mxCuBE v1.3 plugin working directory: strDir = os.path.dirname(self.getWorkingDirectory()) strPath = os.path.join(strDir, "bestfile.par") EDUtilsFile.writeFile(strPath, strBestfilePar) break # Execute plugin which creates a simple HTML page self.executeSimpleHTML(xsDataResultCharacterisation) # Upload the best wilson plot path to ISPyB strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation) strBestWilsonPlotPyarchPath = None if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None: # Copy wilson path to Pyarch strBestWilsonPlotPyarchPath = os.path.join( strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath) ) if not os.path.exists(strBestWilsonPlotPyarchPath): if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)): os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755) shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath) self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath) if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None: xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath() xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = ( self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId ) xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath) edPluginSetBestWilsonPlotPath = self.loadPlugin( "EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath" ) edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath edPluginSetBestWilsonPlotPath.executeSynchronous() # Only for the ESRF: if EDUtilsPath.isESRF(): # For EXI: create workflow entry with one workflow step self.edPluginStoreWorkflow = self.loadPlugin(self.strPluginStoreWorkflow) self.edPluginStoreWorkflowStep = self.loadPlugin(self.strPluginStoreWorkflowStep) self.edPluginUpdateDataCollectionGroupWorkflowId = self.loadPlugin( self.strUpdateDataCollectionGroupWorkflowId ) xsDataISPyBWorkflow = XSDataISPyBWorkflow() xsDataISPyBWorkflow.workflowType = XSDataString("Characterisation") xsDataISPyBWorkflow.workflowTitle = XSDataString("Characterisation") xsDataInputISPyBStoreWorkflow = XSDataInputISPyBStoreWorkflow() xsDataInputISPyBStoreWorkflow.workflow = xsDataISPyBWorkflow self.edPluginStoreWorkflow.dataInput = xsDataInputISPyBStoreWorkflow self.edPluginStoreWorkflow.executeSynchronous() if ( self.edPluginStoreWorkflow.dataOutput is not None and self.edPluginStoreWorkflow.dataOutput.workflowId is not None ): workflowId = self.edPluginStoreWorkflow.dataOutput.workflowId.value # Update data collection group xsDataInputISPyBUpdateDataCollectionGroupWorkflowId = ( XSDataInputISPyBUpdateDataCollectionGroupWorkflowId() ) xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.workflowId = XSDataInteger(workflowId) xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileLocation = XSDataString( os.path.dirname(self.xsDataFirstImage.path.value) ) xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileName = XSDataString( os.path.basename(self.xsDataFirstImage.path.value) ) self.edPluginUpdateDataCollectionGroupWorkflowId.dataInput = ( xsDataInputISPyBUpdateDataCollectionGroupWorkflowId ) self.edPluginUpdateDataCollectionGroupWorkflowId.executeSynchronous() xsDataInputISPyBStoreWorkflowStep = XSDataInputISPyBStoreWorkflowStep() xsDataInputISPyBStoreWorkflowStep.workflowId = XSDataInteger(workflowId) xsDataInputISPyBStoreWorkflowStep.workflowStepType = XSDataString("Characterisation") xsDataInputISPyBStoreWorkflowStep.status = XSDataString("Success") if strBestWilsonPlotPyarchPath is not None: xsDataInputISPyBStoreWorkflowStep.imageResultFilePath = XSDataString( strBestWilsonPlotPyarchPath ) xsDataInputISPyBStoreWorkflowStep.htmlResultFilePath = XSDataString(strPyArchPathToDNAFileDirectory) if self.edPluginExecSimpleHTML.dataOutput is not None: strResultFilePath = self.edPluginExecSimpleHTML.dataOutput.pathToJsonFile.path.value # strPyarchResultFilePath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strResultFilePath) xsDataInputISPyBStoreWorkflowStep.resultFilePath = XSDataString(strResultFilePath) self.edPluginStoreWorkflowStep.dataInput = xsDataInputISPyBStoreWorkflowStep self.edPluginStoreWorkflowStep.executeSynchronous()
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlDozorv1_0.postProcess") # Write a file to be used with ISPyB or GNUPLOT only if data collection id in input dataCollectionId = None if self.dataInput.dataCollectionId is None and len(self.dataInput.image) > 0: # Only try to obtain data collection id if at the ESRF and path starts with "/data" if EDUtilsPath.isESRF() and self.dataInput.image[0].path.value.startswith("/data"): xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() xsDataInputRetrieveDataCollection.image = XSDataImage(self.dataInput.image[0].path) edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4") edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection edPluginISPyBRetrieveDataCollection.executeSynchronous() xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput if xsDataResultRetrieveDataCollection is not None: dataCollection = xsDataResultRetrieveDataCollection.dataCollection if dataCollection is not None: dataCollectionId = dataCollection.dataCollectionId elif self.dataInput.dataCollectionId is not None: dataCollectionId = self.dataInput.dataCollectionId.value if dataCollectionId is not None: minImageNumber = None maxImageNumber = None minAngle = None maxAngle = None minDozorValue = None maxDozorValue = None minResolution = None maxResolution = None dozorPlotFileName = "dozor_{0}.png".format(dataCollectionId) dozorCsvFileName = "dozor_{0}.csv".format(dataCollectionId) with open(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), "w") as gnuplotFile: gnuplotFile.write("# Data directory: {0}\n".format(self.directory)) gnuplotFile.write("# File template: {0}\n".format(self.template.replace("%04d", "####"))) gnuplotFile.write("# {0:>9s}{1:>16s}{2:>16s}{3:>16s}{4:>16s}{5:>16s}\n".format("'Image no'", "'Angle'", "'No of spots'", "'Main score (*10)'", "'Spot score'", "'Visible res.'", )) for imageDozor in self.dataOutput.imageDozor: gnuplotFile.write("{0:10d},{1:15.3f},{2:15d},{3:15.3f},{4:15.3f},{5:15.3f}\n".format(imageDozor.number.value, imageDozor.angle.value, imageDozor.spotsNumOf.value, 10 * imageDozor.mainScore.value, imageDozor.spotScore.value, imageDozor.visibleResolution.value, )) if minImageNumber is None or minImageNumber > imageDozor.number.value: minImageNumber = imageDozor.number.value minAngle = imageDozor.angle.value if maxImageNumber is None or maxImageNumber < imageDozor.number.value: maxImageNumber = imageDozor.number.value maxAngle = imageDozor.angle.value if minDozorValue is None or minDozorValue > imageDozor.mainScore.value: minDozorValue = imageDozor.spotScore.value if maxDozorValue is None or maxDozorValue < imageDozor.mainScore.value: maxDozorValue = imageDozor.spotScore.value # Min resolution: the higher the value the lower the resolution if minResolution is None or minResolution < imageDozor.visibleResolution.value: # Disregard resolution worse than 10.0 if imageDozor.visibleResolution.value < 10.0: minResolution = imageDozor.visibleResolution.value # Max resolution: the lower the number the better the resolution if maxResolution is None or maxResolution > imageDozor.visibleResolution.value: maxResolution = imageDozor.visibleResolution.value xtics = "" if minImageNumber is not None and minImageNumber == maxImageNumber: minAngle -= 1.0 maxAngle += 1.0 noImages = maxImageNumber - minImageNumber + 1 if noImages <= 4: minImageNumber -= 0.1 maxImageNumber += 0.1 deltaAngle = maxAngle - minAngle minAngle -= deltaAngle * 0.1 / noImages maxAngle += deltaAngle * 0.1 / noImages xtics = "1" if maxResolution is None or maxResolution > 0.8: maxResolution = 0.8 else: maxResolution = int(maxResolution * 10.0) / 10.0 if minResolution is None or minResolution < 4.5: minResolution = 4.5 else: minResolution = int(minResolution * 10.0) / 10.0 + 1 if maxDozorValue < 0.001 and minDozorValue < 0.001: yscale = "set yrange [-0.5:0.5]\n set ytics 1" else: yscale = "set autoscale y" gnuplotFile.close() gnuplotScript = \ """# set terminal png set output '{dozorPlotFileName}' set title '{title}' set grid x2 y2 set xlabel 'Image number' set x2label 'Angle (degrees)' set y2label 'Resolution (A)' set ylabel 'Number of spots / Dozor score (*10)' set xtics {xtics} nomirror set x2tics set ytics nomirror set y2tics set xrange [{minImageNumber}:{maxImageNumber}] set x2range [{minAngle}:{maxAngle}] {yscale} set y2range [{minResolution}:{maxResolution}] set key below plot '{dozorCsvFileName}' using 1:3 title 'Number of spots' axes x1y1 with points linetype rgb 'goldenrod' pointtype 7 pointsize 1.5, \ '{dozorCsvFileName}' using 1:4 title 'Dozor score' axes x1y1 with points linetype 3 pointtype 7 pointsize 1.5, \ '{dozorCsvFileName}' using 1:6 title 'Visible resolution' axes x1y2 with points linetype 1 pointtype 7 pointsize 1.5 """.format(title=self.template.replace("%04d", "####"), dozorPlotFileName=dozorPlotFileName, dozorCsvFileName=dozorCsvFileName, minImageNumber=minImageNumber, maxImageNumber=maxImageNumber, minAngle=minAngle, maxAngle=maxAngle, minResolution=minResolution, maxResolution=maxResolution, xtics=xtics, yscale=yscale, ) pathGnuplotScript = os.path.join(self.getWorkingDirectory(), "gnuplot.sh") data_file = open(pathGnuplotScript, "w") data_file.write(gnuplotScript) data_file.close() oldCwd = os.getcwd() os.chdir(self.getWorkingDirectory()) os.system("{0} {1}".format(self.gnuplot, pathGnuplotScript)) os.chdir(oldCwd) self.dataOutput.dozorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), dozorPlotFileName))) if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = os.path.join(self.directory.replace("RAW_DATA", "PROCESSED_DATA"), "DozorPlot") resultsDirectory = os.path.join(processDirectory, "results") dozorPlotResultPath = os.path.join(resultsDirectory, dozorPlotFileName) dozorCsvResultPath = os.path.join(resultsDirectory, dozorCsvFileName) try: if not os.path.exists(resultsDirectory): os.makedirs(resultsDirectory, 0o755) shutil.copy(os.path.join(self.getWorkingDirectory(), dozorPlotFileName), dozorPlotResultPath) shutil.copy(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), dozorCsvResultPath) except: self.warning("Couldn't copy files to results directory: {0}".format(resultsDirectory)) try: # Create paths on pyarch dozorPlotPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorPlotResultPath) dozorCsvPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorCsvResultPath) if not os.path.exists(os.path.dirname(dozorPlotPyarchPath)): os.makedirs(os.path.dirname(dozorPlotPyarchPath), 0o755) shutil.copy(dozorPlotResultPath, dozorPlotPyarchPath) shutil.copy(dozorCsvResultPath, dozorCsvPyarchPath) # Upload to data collection xsDataInputISPyBSetImageQualityIndicatorsPlot = XSDataInputISPyBSetImageQualityIndicatorsPlot() xsDataInputISPyBSetImageQualityIndicatorsPlot.dataCollectionId = XSDataInteger(dataCollectionId) xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsPlotPath = XSDataString(dozorPlotPyarchPath) xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsCSVPath = XSDataString(dozorCsvPyarchPath) EDPluginISPyBSetImageQualityIndicatorsPlot = self.loadPlugin("EDPluginISPyBSetImageQualityIndicatorsPlotv1_4") EDPluginISPyBSetImageQualityIndicatorsPlot.dataInput = xsDataInputISPyBSetImageQualityIndicatorsPlot EDPluginISPyBSetImageQualityIndicatorsPlot.executeSynchronous() except: self.warning("Couldn't copy files to pyarch: {0}".format(dozorPlotPyarchPath)) self.sendMessageToMXCuBE("Processing finished", "info") self.setStatusToMXCuBE("Success")
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.dataInput.diffractionImage.path.value strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1] if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]: self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the MXWaitFile plugin xsDataInputMXWaitFile = XSDataInputMXWaitFile() pathToImageFile = strPathToDiffractionImage # Quite ugly hack to avoid lag problems at the ESRF: if EDUtilsPath.isESRF(): if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]): # Pilatus 6M self.minImageSize = 6000000 elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]): # Pilatus3 2M self.minImageSize = 2000000 elif strImageFileNameExtension == ".h5": self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile) pathToImageFile = self.h5DataFilePath self.isH5 = True elif EDUtilsPath.isEMBL(): self.minImageSize = 10000 xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile))) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut()) self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) # Load the execution plugin self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName) xsDataInputMXThumbnail = XSDataInputMXThumbnail() xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage() xsDataInputMXThumbnail.height = XSDataInteger(1024) xsDataInputMXThumbnail.width = XSDataInteger(1024) xsDataInputMXThumbnail.format = self.dataInput.format # Output path strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except Exception as e: self.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: self.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0755) strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser) os.chmod(strOutputDirname, 0755) self.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt) if self.dataInput.format is not None: self.strSuffix = self.dataInput.format.value.lower() self.strImageFormat = self.dataInput.format.value.upper() self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix) xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath))) self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection( ) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace( "%05d", "####") else: template = ispybDataCollection.fileTemplate.replace( "%04d", "####") if self.dataInput.startFrame is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.startFrame.value if self.dataInput.endFrame is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.endFrame.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoEnd) # else: # directory = self.dataInput.dirN.value # template = self.dataInput.templateN.value # imageNoStart = self.dataInput.fromN.value # imageNoEnd = self.dataInput.toN.value # fileTemplate = template.replace("####", "%04d") # pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) # pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath( beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace( 'PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isEMBL(): fWaitFileTimeout = 60 else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile( XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % ( fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() # Prepare input to execution plugin if self.doAnom: xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS() xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True) xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSAnom.startFrame = XSDataInteger( imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd) if self.doNoanom: xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS() xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False) xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger( imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd) if isH5: masterFilePath = os.path.join( directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(masterFilePath))) if self.doNoanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(masterFilePath))) else: if self.doAnom: xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(pathToStartImage))) if self.doNoanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(pathToStartImage))) self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) if self.doAnom: self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom self.edPluginExecXia2DIALSAnom.execute() if self.doNoanom: self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom self.edPluginExecXia2DIALSNoanom.execute() if self.doAnom: self.edPluginExecXia2DIALSAnom.synchronize() if self.doNoanom: self.edPluginExecXia2DIALSNoanom.synchronize() self.timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSAnom, True, proposal, self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doNoanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSNoanom, False, proposal, self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")