def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlReadImageHeaderv10.preProcess") # First determine the type of image xsDataInputReadImageHeader = self.getDataInput() xsDataFileImage = xsDataInputReadImageHeader.getImage() self.strFileImagePath = xsDataFileImage.getPath().getValue() # Plugin for waiting for files self.edPluginExecMXWaitFile = self.loadPlugin( self.strPluginExecMXWaitFile) xsDataInputMXWaitFile = XSDataInputMXWaitFile() xsDataInputMXWaitFile.setFile( XSDataFile(XSDataString(self.strFileImagePath))) xsDataInputMXWaitFile.setSize(XSDataInteger(100000)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.edPluginExecMXWaitFile.setDataInput(xsDataInputMXWaitFile)
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlReadImageHeaderv10.preProcess") # First determine the type of image xsDataInputReadImageHeader = self.getDataInput() xsDataFileImage = xsDataInputReadImageHeader.getImage() self.strFileImagePath = xsDataFileImage.getPath().getValue() # Plugin for waiting for files self.edPluginExecMXWaitFile = self.loadPlugin(self.strPluginExecMXWaitFile) xsDataInputMXWaitFile = XSDataInputMXWaitFile() xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(self.strFileImagePath))) xsDataInputMXWaitFile.setSize(XSDataInteger(100000)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.edPluginExecMXWaitFile.setDataInput(xsDataInputMXWaitFile)
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.dataInput.diffractionImage.path.value strImageFileNameExtension = os.path.splitext( strPathToDiffractionImage)[1] if not strImageFileNameExtension in [ ".img", ".marccd", ".mccd", ".cbf", ".h5" ]: self.error( "Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the MXWaitFile plugin xsDataInputMXWaitFile = XSDataInputMXWaitFile() pathToImageFile = strPathToDiffractionImage # Quite ugly hack to avoid lag problems at the ESRF: if EDUtilsPath.isESRF() or EDUtilsPath.isALBA(): if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]): # Pilatus 6M self.minImageSize = 6000000 elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]): # Pilatus3 2M self.minImageSize = 2000000 elif strImageFileNameExtension == ".h5": self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath( pathToImageFile) pathToImageFile = self.h5DataFilePath self.isH5 = True elif EDUtilsPath.isEMBL(): self.minImageSize = 10000 xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setFile( XSDataFile(XSDataString(pathToImageFile))) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputMXWaitFile.setTimeOut( self.getDataInput().getWaitForFileTimeOut()) self.edPluginMXWaitFile = self.loadPlugin( self.strMXWaitFilePluginName) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) # Load the execution plugin self.edPluginExecThumbnail = self.loadPlugin( self.strExecThumbnailPluginName) xsDataInputMXThumbnail = XSDataInputMXThumbnail() xsDataInputMXThumbnail.image = self.getDataInput( ).getDiffractionImage() xsDataInputMXThumbnail.height = XSDataInteger(1024) xsDataInputMXThumbnail.width = XSDataInteger(1024) xsDataInputMXThumbnail.format = self.dataInput.format # Output path strImageNameWithoutExt = os.path.basename( os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput( ).getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join( strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath( strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except Exception as e: self.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: self.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0o755) strOutputDirname = tempfile.mkdtemp( prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser) os.chmod(strOutputDirname, 0o755) self.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join( strOutputDirname, strImageNameWithoutExt) if self.dataInput.format is not None: self.strSuffix = self.dataInput.format.value.lower() self.strImageFormat = self.dataInput.format.value.upper() self.strOutputPath = os.path.join( self.strOutputPathWithoutExtension + "." + self.strSuffix) xsDataInputMXThumbnail.setOutputPath( XSDataFile(XSDataString(self.strOutputPath))) self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(5000000)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() edPluginControlBackground3D = self.loadPlugin( self.strPluginNameControlBackground3D) listPlugin.append([ edPluginPluginExecImageQualityIndicator, edPluginControlBackground3D ]) xsDataInputControlBackground3D = XSDataInputControlBackground3D( ) xsDataInputControlBackground3D.addImage( XSDataFile(xsDataImage.path)) edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D edPluginControlBackground3D.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for pluginPair in listPlugin: edPluginPluginExecImageQualityIndicator = pluginPair[0] edPluginControlBackground3D = pluginPair[1] edPluginPluginExecImageQualityIndicator.synchronize() edPluginControlBackground3D.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) if edPluginControlBackground3D.dataOutput.imageBackground != []: xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[ 0].estimate self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have background3D_estimate from all images: has_background3D_estimate = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.background3D_estimate is None: has_background3D_estimate = False if has_background3D_estimate: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. background3D_estimate.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) listPlugin.append(edPluginPluginExecImageQualityIndicator) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for edPluginPluginExecImageQualityIndicator in listPlugin: edPluginPluginExecImageQualityIndicator.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.bDoISPyBUpload: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key= lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value ) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXDSAPPv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" if self.dataInput.startImageNumber is not None: imageNoStart = self.dataInput.startImageNumber.value if self.dataInput.endImageNumber is not None: imageNoEnd = self.dataInput.endImageNumber.value # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") if imageNoStart is None: imageNoStart = ispybDataCollection.startImageNumber if imageNoEnd is None: imageNoEnd = ispybDataCollection.startImageNumber + \ ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "XDSAPP", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() if self.dataInput.hdf5ToCbfDirectory is not None: dir = self.dataInput.hdf5ToCbfDirectory.path.value pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0] self.timeStart = time.localtime() # Prepare input to execution plugin if self.doAnom: xsDataInputXDSAPPAnom = XSDataInputXDSAPP() xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True) xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup) self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom self.edPluginExecXDSAPPAnom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doNoanom: xsDataInputXDSAPPNoanom = XSDataInputXDSAPP() xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False) xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup) self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom self.edPluginExecXDSAPPNoanom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) if self.doAnom: self.edPluginExecXDSAPPAnom.synchronize() xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput # Run XSCALE even if XSCALE.LP is present strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True) if self.doNoanom: self.edPluginExecXDSAPPNoanom.synchronize() xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True) self.timeEnd = time.localtime() # Upload to ISPyB if self.dataInput.dataCollectionId is not None: # Check if we should use XDS_ASCII_to_XML.pl if self.doAnom: if self.useXdsAsciiToXml: # Only for anom runs self.runXdsAsciiToXml(xsDataResultXDSAPPAnom, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) else: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template, strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doNoanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template, strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection( ) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace( "%05d", "####") else: template = ispybDataCollection.fileTemplate.replace( "%04d", "####") if self.dataInput.startFrame is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.startFrame.value if self.dataInput.endFrame is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.endFrame.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoEnd) # else: # directory = self.dataInput.dirN.value # template = self.dataInput.templateN.value # imageNoStart = self.dataInput.fromN.value # imageNoEnd = self.dataInput.toN.value # fileTemplate = template.replace("####", "%04d") # pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) # pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath( beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace( 'PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isEMBL(): fWaitFileTimeout = 60 else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile( XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % ( fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() # Prepare input to execution plugin if self.doAnom: xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS() xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True) xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSAnom.startFrame = XSDataInteger( imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd) if self.doNoanom: xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS() xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False) xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger( imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd) if isH5: masterFilePath = os.path.join( directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(masterFilePath))) if self.doNoanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(masterFilePath))) else: if self.doAnom: xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(pathToStartImage))) if self.doNoanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(pathToStartImage))) self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) if self.doAnom: self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom self.edPluginExecXia2DIALSAnom.execute() if self.doNoanom: self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom self.edPluginExecXia2DIALSNoanom.execute() if self.doAnom: self.edPluginExecXia2DIALSAnom.synchronize() if self.doNoanom: self.edPluginExecXia2DIALSNoanom.synchronize() self.timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSAnom, True, proposal, self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doNoanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSNoanom, False, proposal, self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXDSAPPv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() if self.dataInput.hdf5ToCbfDirectory is not None: dir = self.dataInput.hdf5ToCbfDirectory.path.value pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0] # Prepare input to execution plugin xsDataInputXDSAPPAnom = XSDataInputXDSAPP() xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True) xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup) if self.doAnomAndNonanom: xsDataInputXDSAPPNoanom = XSDataInputXDSAPP() xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False) xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup) # if isH5: # masterFilePath = os.path.join(directory, # self.eiger_template_to_master(template)) # xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(masterFilePath))) # if self.doAnomAndNonanom: # xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(masterFilePath))) # else: # xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(pathToStartImage))) # if self.doAnomAndNonanom: # xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(pathToStartImage))) self.timeStart = time.localtime() self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom self.edPluginExecXDSAPPAnom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doAnomAndNonanom: self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom self.edPluginExecXDSAPPNoanom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) self.edPluginExecXDSAPPAnom.synchronize() xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput # Run XSCALE even if XSCALE.LP is present strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True) if self.doAnomAndNonanom: self.edPluginExecXDSAPPNoanom.synchronize() xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True) self.timeEnd = time.localtime() # Upload to ISPyB if self.dataInput.dataCollectionId is not None: # Check if we should use XDS_ASCII_to_XML.pl if self.useXdsAsciiToXml: # Only for anom runs self.runXdsAsciiToXml(xsDataResultXDSAPPAnom, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) else: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template, strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doAnomAndNonanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template, strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlAutoPROCv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5) elif EDUtilsPath.isMAXIV(): template = ispybDataCollection.fileTemplate else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") if self.dataInput.fromN is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.fromN.value if self.dataInput.toN is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.toN.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.path.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value if EDUtilsPath.isEMBL(): fileTemplate = template.replace("#####", "%05d") else: fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory if EDUtilsPath.isALBA(): _processDirectory = "_".join(pathToStartImage.split('_')[:-1]) from datetime import datetime _id = datetime.now().strftime('%Y%m%d_%H%M%S') self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id) else: self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "autoPROC", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # The resultsDirectory is not used at ALBA (only pyarchDirectory) if EDUtilsPath.isALBA(): self.resultsDirectory = None # Determine pyarch prefix if EDUtilsPath.isALBA(): listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]), listPrefix[-2]) else: listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isMAXIV(): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV(): fWaitFileTimeout = 60 # s else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=True, timeStart=self.timeStart) self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=False, timeStart=self.timeStart) self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=False, timeStart=self.timeStart) # Prepare input to execution plugin if self.doAnom: xsDataInputAutoPROCAnom = XSDataInputAutoPROC() xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True) xsDataInputAutoPROCAnom.symm = self.dataInput.symm xsDataInputAutoPROCAnom.cell = self.dataInput.cell xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit if self.doNoanom: xsDataInputAutoPROCNoanom = XSDataInputAutoPROC() xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False) xsDataInputAutoPROCNoanom.symm = self.dataInput.symm xsDataInputAutoPROCNoanom.cell = self.dataInput.cell xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier() xsDataAutoPROCIdentifier.idN = XSDataString(identifier) xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory)) xsDataAutoPROCIdentifier.templateN = XSDataString(template) xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart) xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd) if self.doAnom: xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier) if self.doNoanom: xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy()) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath)) if self.doNoanom: xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath)) timeStart = time.localtime() if self.doAnom: self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom self.edPluginExecAutoPROCAnom.execute() if self.doNoanom: self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom self.edPluginExecAutoPROCNoanom.execute() if self.doAnom: self.edPluginExecAutoPROCAnom.synchronize() if self.doNoanom: self.edPluginExecAutoPROCNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd) if self.doNoanom: self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def process(self, _edPlugin=None): EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_4.process...") xsDataInputMXCuBE = self.getDataInput() xsDataInputInterface = XSDataInputInterface() self.edPluginControlInterface = self.loadPlugin(self.strPluginControlInterface) self.xsDataFirstImage = None for xsDataSetMXCuBE in xsDataInputMXCuBE.getDataSet(): for xsDataFile in xsDataSetMXCuBE.getImageFile(): if xsDataFile.path.value.endswith(".h5"): self.bIsEigerDetector = True self.xsDataFirstImage = xsDataFile xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF() xsDataInputControlH5ToCBF.hdf5File = XSDataFile(xsDataFile.path) xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(EDUtilsImage.getImageNumber(xsDataFile.path.value)) edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF") edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF edPluginControlH5ToCBF.executeSynchronous() cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile xsDataInputInterface.addImagePath(cbfFile) else: xsDataInputInterface.addImagePath(xsDataFile) if self.xsDataFirstImage is None: self.xsDataFirstImage = xsDataFile # Check that images are on disk, otherwise wait for xsDataImagePath in xsDataInputInterface.imagePath: imagePath = xsDataImagePath.path.value if os.path.exists(imagePath): message = "Input file ok: {0}".format(imagePath) self.screen(message) self.sendMessageToMXCuBE(message) else: self.sendMessageToMXCuBE("Waiting for file: {0}, timeout {1} s".format(imagePath, self.fMXWaitFileTimeOut)) edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile = XSDataInputMXWaitFile() xsDataInputMXWaitFile.file = XSDataFile(XSDataString(imagePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) edPluginMXWaitFile.executeSynchronous() if edPluginMXWaitFile.dataOutput.timedOut.value: errorMessage = "ERROR! File {0} does not exist on disk.".format(imagePath) self.ERROR(errorMessage) self.sendMessageToMXCuBE(errorMessage, "error") self.setFailure() break xsDataExperimentalCondition = self.getFluxAndBeamSizeFromISPyB(self.xsDataFirstImage, \ xsDataInputMXCuBE.getExperimentalCondition()) xsDataInputInterface.setExperimentalCondition(xsDataExperimentalCondition) xsDataInputInterface.setDiffractionPlan(xsDataInputMXCuBE.getDiffractionPlan()) xsDataInputInterface.setSample(xsDataInputMXCuBE.getSample()) xsDataInputInterface.setDataCollectionId(xsDataInputMXCuBE.getDataCollectionId()) self.edPluginControlInterface.setDataInput(xsDataInputInterface) if not self.isFailure() and self.edPluginControlInterface is not None: self.connectProcess(self.edPluginControlInterface.executeSynchronous) self.edPluginControlInterface.connectSUCCESS(self.doSuccessActionInterface) self.edPluginControlInterface.connectFAILURE(self.doFailureActionInterface)
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listBatch = [] indexBatch = 0 listH5FilePath = [] ispybDataCollection = None for xsDataImage in listXSDataImage: strPathToImage = xsDataImage.path.value # If Eiger, just wait for the h5 file if "id30a3" in strPathToImage: h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") ispybDataCollection = None time.sleep(1) indexLoop = 1 continueLoop = True while continueLoop: xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF() xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage)) imageNumber = EDUtilsImage.getImageNumber(strPathToImage) xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber) xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber) xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop)) edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF edPluginControlH5ToCBF.executeSynchronous() cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile # print(cbfFile) # print(indexLoop) if cbfFile is not None: strPathToImage = cbfFile.path.value # print(cbfFile.path.value) if os.path.exists(strPathToImage): self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection elif not os.path.exists(strPathToImage): self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() else: # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) listBatch.append(xsDataImageNew.copy()) if len(listBatch) == batchSize: edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, listBatch)) listBatch = [] if len(listBatch) > 0: # Process the remaining images... edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append([edPluginControlDozor, listBatch]) listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator]) edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen( "Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath( strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen( "ID30a3 Eiger data, waiting for master and data files..." ) listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen( "Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile( listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger( firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger( lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile( XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin( "EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:06d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join( directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace( ".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber( image.path.value) oldPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:04d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) )) strPathToImage = os.path.join( directory, lastCbfFile) self.screen( "Image has been converted to CBF file: {0}" .format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen( "Still waiting for converting to CBF file: {0}" .format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger( EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage( xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append( (xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_{0}".format( os.path.splitext( os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append( (edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen( "No dozor results! Re-executing Dozor for images {0} to {1}" .format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format( os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile( image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists( imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt( imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString( base64.b64encode( numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput( ).parseString( edPluginControlDozor.dataOutput. inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[ -5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader( ) xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path ) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin( self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing( ) xsDataInputLabelitIndexing.image.append( XSDataImage( xsDataResultControlImageQualityIndicator. image.path)) edPluginLabelitIndexing.setDataInput( xsDataInputLabelitIndexing) self.listPluginLabelit.append([ edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator ]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure( ) and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput( ) xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult( xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlDozorv1_1.process") msg = "EDPluginControlDozorXmlRpcv1_0 started..." self.sendMessageToMXCuBE("Processing started...", "info") xsDataResultControlDozor = XSDataResultControlDozor() firstFileName = self.dataInput.template.value % ( self.dataInput.first_run_number.value, self.dataInput.first_image_number.value) fWaitFileTimeout = 180 #sec self.waitFileFirst = self.loadPlugin("EDPluginMXWaitFilev1_1", "MXWaitFileFirst") xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(firstFileName)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.waitFileFirst.size = XSDataInteger(1000000) self.waitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.waitFileFirst.executeSynchronous() if self.waitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, firstFileName) #self.addWarningMessage(strWarningMessage) #self.WARNING(strWarningMessage) #self.sendMessageToMXCuBE(strWarningMessage, "error") edPluginControlReadImageHeader = self.loadPlugin( self.strEDPluginControlReadImageHeaderName) xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( XSDataString(firstFileName)) edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader edPluginControlReadImageHeader.executeSynchronous() subWedge = edPluginControlReadImageHeader.dataOutput.subWedge xsDataInputDozor = XSDataInputDozor() beam = subWedge.experimentalCondition.beam detector = subWedge.experimentalCondition.detector goniostat = subWedge.experimentalCondition.goniostat xsDataInputDozor.detectorType = detector.type xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value) xsDataInputDozor.spotSize = XSDataDouble(3.0) xsDataInputDozor.detectorDistance = XSDataDouble( detector.distance.value) xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value) orgx = detector.beamPositionY.value / detector.pixelSizeY.value orgy = detector.beamPositionX.value / detector.pixelSizeX.value xsDataInputDozor.orgx = XSDataDouble(orgx) xsDataInputDozor.orgy = XSDataDouble(orgy) # GB: the 50 might need tunig to CPU speed and number of. 2000 is a current limit of Dozor. self.maxChunkSize = 4000 #a min (150 * 2527 * 2463 /detector.numberPixelX.value / detector.numberPixelY.value, 2000) _beamstop = self.beamstop(detector) if _beamstop is not None: #self.WARNING("Setting beamstop shadow: %s"%_beamstop) xsDataInputDozor.ixMin = XSDataInteger(_beamstop['ix_min']) xsDataInputDozor.iyMin = XSDataInteger(_beamstop['iy_min']) xsDataInputDozor.ixMax = XSDataInteger(_beamstop['ix_max']) xsDataInputDozor.iyMax = XSDataInteger(_beamstop['iy_max']) if self.dataInput.pixelMin is not None: xsDataInputDozor.pixelMin = self.dataInput.pixelMin if self.dataInput.pixelMax is not None: xsDataInputDozor.pixelMax = self.dataInput.pixelMax _serial = 0 _startTime = time.time() chunk_list = self.schedule(goniostat.rotationAxisStart.value, goniostat.oscillationWidth.value) for chunk in chunk_list: if not self.poll_file( self.dataInput.template.value % (chunk['run_number'], chunk['first'] + chunk['number_of'] - 1), (beam.exposureTime.value + 0.003) * chunk['number_of'] + 30): self.sendMessageToMXCuBE("Timeout waiting for frame: %d" % (chunk['first'] + chunk['number_of']), level="error") return xsDataInputDozor.oscillationRange = XSDataDouble( chunk['rotation_range']) xsDataInputDozor.startingAngle = XSDataDouble( chunk['rotation_start']) xsDataInputDozor.firstImageNumber = XSDataInteger(chunk['first']) xsDataInputDozor.numberImages = XSDataInteger(chunk['number_of']) strFileName = self.dataInput.template.value % (chunk['run_number'], chunk['first']) strXDSTemplate = EDUtilsImage.getTemplate(strFileName, '?') xsDataInputDozor.nameTemplateImage = XSDataString( os.path.join(os.path.dirname(strFileName), strXDSTemplate)) edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor") edPluginDozor.dataInput = xsDataInputDozor edPluginDozor.executeSynchronous() xsDataChunkResultControlImageDozor = XSDataResultControlDozor() dozor_batch_list = [] #[[1,4,3,8,6], #[2, 5, 3, 3, 7],[1, 3, 12, 11, 4],[4, 2, 3, 5, 7]] #ozor_image_dict = {} diff_image_count = 0 xsDataControlImageDozor = XSDataControlImageDozor() for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor: xsDataControlImageDozor = XSDataControlImageDozor() xsDataControlImageDozor.number = xsDataResultDozor.number strFileName = self.dataInput.template.value % ( chunk['run_number'], xsDataControlImageDozor.number.value) xsDataControlImageDozor.image = XSDataFile( XSDataString(strFileName)) xsDataControlImageDozor.spots_num_of = xsDataResultDozor.spots_num_of xsDataControlImageDozor.spots_int_aver = xsDataResultDozor.spots_int_aver xsDataControlImageDozor.spots_resolution = xsDataResultDozor.spots_resolution xsDataControlImageDozor.powder_wilson_scale = xsDataResultDozor.powder_wilson_scale xsDataControlImageDozor.powder_wilson_bfactor = xsDataResultDozor.powder_wilson_bfactor xsDataControlImageDozor.powder_wilson_resolution = xsDataResultDozor.powder_wilson_resolution xsDataControlImageDozor.powder_wilson_correlation = xsDataResultDozor.powder_wilson_correlation xsDataControlImageDozor.powder_wilson_rfactor = xsDataResultDozor.powder_wilson_rfactor xsDataControlImageDozor.score = xsDataResultDozor.score xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor) dozor_batch_list.append( (xsDataControlImageDozor.number.getValue(), xsDataControlImageDozor.spots_num_of.getValue(), xsDataControlImageDozor.spots_int_aver.getValue(), xsDataControlImageDozor.spots_resolution.getValue(), xsDataControlImageDozor.score.getValue())) if xsDataControlImageDozor.spots_num_of.getValue() > 0: diff_image_count += 1 xsDataChunkResultControlImageDozor.addImageDozor( xsDataControlImageDozor) xsDataChunkResultControlImageDozor.exportToFile( "ResultControlDozor_Chunk_%06d.xml" % _serial) self.sendResultToMXCuBE(dozor_batch_list) _serial += 1 self.screen("Chunk %d/%d done in %.3f seconds" % \ (_serial, len(chunk_list), time.time()-_startTime)) self.sendMessageToMXCuBE("Chunk %d/%d done in %.2f sec., num diffr. frames: %d/%d" % \ (_serial, len(chunk_list), time.time() - _startTime, diff_image_count, len(dozor_batch_list) ) ) _startTime = time.time()
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile(listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger(firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger(lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace(".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber(image.path.value) oldPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger(EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_{0}".format(os.path.splitext(os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen("No dozor results! Re-executing Dozor for images {0} to {1}".format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format(os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing() xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path)) edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing) self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput() xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0755) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() # Prepare input to execution plugin xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS() xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS() xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath))) else: xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage))) timeStart = time.localtime() self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom self.edPluginExecXia2DIALSAnom.execute() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom self.edPluginExecXia2DIALSNoanom.execute() self.edPluginExecXia2DIALSAnom.synchronize() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd) if self.doAnomAndNonanom: self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listBatch = [] indexBatch = 0 for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(5000000)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: # Check if we should run distl.signalStrength edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator)) listBatch.append(xsDataImage.copy()) if len(listBatch) == batchSize: edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, listBatch)) listBatch = [] if len(listBatch) > 0: # Process the remaining images... edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append([edPluginControlDozor, listBatch]) listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing() xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path)) edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing) self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput() xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.dataInput.diffractionImage.path.value strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1] if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]: self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the MXWaitFile plugin xsDataInputMXWaitFile = XSDataInputMXWaitFile() pathToImageFile = strPathToDiffractionImage # Quite ugly hack to avoid lag problems at the ESRF: if EDUtilsPath.isESRF(): if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]): # Pilatus 6M self.minImageSize = 6000000 elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]): # Pilatus3 2M self.minImageSize = 2000000 elif strImageFileNameExtension == ".h5": self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile) pathToImageFile = self.h5DataFilePath self.isH5 = True elif EDUtilsPath.isEMBL(): self.minImageSize = 10000 xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile))) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut()) self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) # Load the execution plugin self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName) xsDataInputMXThumbnail = XSDataInputMXThumbnail() xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage() xsDataInputMXThumbnail.height = XSDataInteger(1024) xsDataInputMXThumbnail.width = XSDataInteger(1024) xsDataInputMXThumbnail.format = self.dataInput.format # Output path strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except Exception as e: self.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: self.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0755) strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser) os.chmod(strOutputDirname, 0755) self.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt) if self.dataInput.format is not None: self.strSuffix = self.dataInput.format.value.lower() self.strImageFormat = self.dataInput.format.value.upper() self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix) xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath))) self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlCrystFELv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB #self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value, # "CrystFEL", # "started") xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection self.ispyb_retrieve_dc_plugin.executeSynchronous() ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): #TODO PE2 has 7 digits others 5 template = ispybDataCollection.fileTemplate.replace("%07d", "####") else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") imageNoStart = ispybDataCollection.startImageNumber imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") try: if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 755) except Exception as ex: print ex # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) #TODO: enable when ready """ if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 755) """ # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 60 xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst) self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst self.wait_first_image_plugin.executeSynchronous() if self.wait_first_image_plugin.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory self.streamFilename = "%s_%d.stream" % ( self.baseName, self.dataInput.dataCollectionId.value ) self.hklFilename = "%s_%d.hkl" % ( self.baseName, self.dataInput.dataCollectionId.value ) self.mtzFilename = "%s_%d.mtz" % ( self.baseName, self.dataInput.dataCollectionId.value ) # Prepare input to execution plugin xsDataInputCrystFEL = XSDataInputCrystFEL() # Check if imagesFullPath exists. If not then generate one if self.dataInput.imagesFullPath is None: imagesFullPath = "%s_images_fullpath.lst" % self.baseName imagesFullFile = open(imagesFullPath, "w") for index in range(imageNoEnd - imageNoStart + 1): imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1)) imagesFullFile.write("%s\n" % imagePath) imagesFullFile.close() xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath) else: xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath xsDataInputCrystFEL.geomFile = self.dataInput.geomFile xsDataInputCrystFEL.cellFile = self.dataInput.cellFile #TODO EDNA can generate the imagesFull path if id do not exist xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename) xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename) xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename) self.timeStart = time.localtime() """ if self.dataInput.dataCollectionId is not None: # Set ISPyB to running self.autoProcIntegrationId, self.autoProcProgramId = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) """ self.index_plugin.dataInput = xsDataInputCrystFEL self.index_plugin.executeSynchronous() if self.index_plugin.isFailure(): self.ERROR('indexamajig: Failed') self.setFailure() return else: self.screen('indexamajig: Finished') self.process_hkl_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL self.partialator_plugin.dataInput = xsDataInputCrystFEL self.post_process_plugin.dataInput = xsDataInputCrystFEL self.process_hkl_plugin_odd.process_hkl_options += " --odd-only" self.process_hkl_plugin_even.process_hkl_options += " --even-only" self.process_hkl_plugin.executeSynchronous() if self.process_hkl_plugin.isFailure(): self.ERROR('process_hkl: Failed') self.setFailure() return else: self.screen('process_hkl: Finished') self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o") self.process_hkl_plugin_odd.executeSynchronous() if self.process_hkl_plugin_odd.isFailure(): self.ERROR('process_hkl_odd: Failed') self.setFailure() return else: self.screen('process_hkl_odd: Finished') self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e") self.process_hkl_plugin_even.executeSynchronous() if self.process_hkl_plugin_even.isFailure(): self.ERROR('process_hkl_even: Failed') self.setFailure() return else: self.screen('process_hkl_even: Finished') self.partialator_plugin.executeSynchronous() if self.partialator_plugin.isFailure(): self.ERROR('partialator: Failed') self.setFailure() return else: self.screen('partialator: Finished') self.post_process_plugin.executeSynchronous() if self.post_process_plugin.isFailure(): self.ERROR('post_process: Failed') self.setFailure() return else: self.screen('post_process: Finished') """ self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL self.edPluginExecCrystFELScale.executeSynchronous() if self.edPluginExecCrystFELScale.isFailure(): self.ERROR('CrystFELScalev1_0: Failed') self.setFailure() return else: self.screen('CrystFELScalev1_0: Finished') self.timeEnd = time.localtime() """ # Upload to ISPyB """