def finallyProcess(self, _edPlugin=None): EDPluginExec.finallyProcess(self, _edPlugin) self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...") xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage() xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName)))) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory()))) # Write workflowStepReport HTML page pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName) pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory()) # Store in Pyarch if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV() or EDUtilsPath.isALBA: strPyarchPath = None if self.xsDataResultCharacterisation is not None: strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection()) if strPyarchPath is None: # For debugging purposes strPyarchPath = EDUtilsPath.getEdnaUserTempFolder() EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath) xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath))) if not os.path.exists(strPyarchPath): os.makedirs(strPyarchPath, 0o755) shutil.copy(pathToJsonFile, strPyarchPath) pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile)) # Write json file xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile)) self.setDataOutput(xsDataResultSimpleHTMLPage)
def eiger_template_to_image(self, fmt, num): fileNumber = int(num / 100) if fileNumber == 0: fileNumber = 1 if EDUtilsPath.isMAXIV(): fmt_string = fmt.replace("%06d", "data_%06d" % fileNumber) else: fmt_string = fmt.replace("####", "1_data_%06d" % fileNumber) return fmt_string.format(num)
def __init__(self): EDLogging.__init__(self) Thread.__init__(self) self.__edSlotPreProcess = EDSlot() self.__edSlotProcess = EDSlot() self.__edSlotPostProcess = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__edSlotFinallyProcess = EDSlot() self.__bIsFailure = False self.__bIsTimeOut = False self.__fTimeOutInSeconds = None if EDUtilsPath.isEMBL() or EDUtilsPath.isALBA() or EDUtilsPath.isMAXIV( ): self.__fDefaultTimeOutInSeconds = 3600.0 else: self.__fDefaultTimeOutInSeconds = 600.0 self.__bIsAbort = False # Reference to the object which calls execute or executeSynchronous self.__edObject = None self.__lExtraTime = [ ] # list of extra allowed time for execution (in second) self.__bLogTiming = False
def eiger_template_to_master(self, fmt): if EDUtilsPath.isMAXIV(): fmt_string = fmt.replace("%06d", "master") else: fmt_string = fmt.replace("####", "1_master") return fmt_string
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlAutoPROCv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5) elif EDUtilsPath.isMAXIV(): template = ispybDataCollection.fileTemplate else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") if self.dataInput.fromN is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.fromN.value if self.dataInput.toN is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.toN.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.path.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value if EDUtilsPath.isEMBL(): fileTemplate = template.replace("#####", "%05d") else: fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory if EDUtilsPath.isALBA(): _processDirectory = "_".join(pathToStartImage.split('_')[:-1]) from datetime import datetime _id = datetime.now().strftime('%Y%m%d_%H%M%S') self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id) else: self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "autoPROC", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # The resultsDirectory is not used at ALBA (only pyarchDirectory) if EDUtilsPath.isALBA(): self.resultsDirectory = None # Determine pyarch prefix if EDUtilsPath.isALBA(): listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]), listPrefix[-2]) else: listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isMAXIV(): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV(): fWaitFileTimeout = 60 # s else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=True, timeStart=self.timeStart) self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=False, timeStart=self.timeStart) self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=False, timeStart=self.timeStart) # Prepare input to execution plugin if self.doAnom: xsDataInputAutoPROCAnom = XSDataInputAutoPROC() xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True) xsDataInputAutoPROCAnom.symm = self.dataInput.symm xsDataInputAutoPROCAnom.cell = self.dataInput.cell xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit if self.doNoanom: xsDataInputAutoPROCNoanom = XSDataInputAutoPROC() xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False) xsDataInputAutoPROCNoanom.symm = self.dataInput.symm xsDataInputAutoPROCNoanom.cell = self.dataInput.cell xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier() xsDataAutoPROCIdentifier.idN = XSDataString(identifier) xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory)) xsDataAutoPROCIdentifier.templateN = XSDataString(template) xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart) xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd) if self.doAnom: xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier) if self.doNoanom: xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy()) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath)) if self.doNoanom: xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath)) timeStart = time.localtime() if self.doAnom: self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom self.edPluginExecAutoPROCAnom.execute() if self.doNoanom: self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom self.edPluginExecAutoPROCNoanom.execute() if self.doAnom: self.edPluginExecAutoPROCAnom.synchronize() if self.doNoanom: self.edPluginExecAutoPROCNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd) if self.doNoanom: self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess") # Check that the input image exists and is of the expected type strPathToDiffractionImage = self.dataInput.diffractionImage.path.value strImageFileNameExtension = os.path.splitext( strPathToDiffractionImage)[1] if not strImageFileNameExtension in [ ".img", ".marccd", ".mccd", ".cbf", ".h5" ]: self.error( "Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage) self.setFailure() else: # Load the MXWaitFile plugin xsDataInputMXWaitFile = XSDataInputMXWaitFile() pathToImageFile = strPathToDiffractionImage # Quite ugly hack to avoid lag problems at the ESRF: if EDUtilsPath.isESRF() or EDUtilsPath.isALBA( ) or EDUtilsPath.isMAXIV(): if any(beamline in strPathToDiffractionImage for beamline in ["id30b"]): # Pilatus 6M self.minImageSize = 6000000 elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]): # Pilatus3 2M self.minImageSize = 2000000 elif strImageFileNameExtension == ".h5": self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath( pathToImageFile) pathToImageFile = self.h5DataFilePath self.isH5 = True elif EDUtilsPath.isEMBL(): self.minImageSize = 10000 xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setFile( XSDataFile(XSDataString(pathToImageFile))) if self.getDataInput().getWaitForFileTimeOut(): xsDataInputMXWaitFile.setTimeOut( self.getDataInput().getWaitForFileTimeOut()) self.edPluginMXWaitFile = self.loadPlugin( self.strMXWaitFilePluginName) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) # Load the execution plugin self.edPluginExecThumbnail = self.loadPlugin( self.strExecThumbnailPluginName) xsDataInputMXThumbnail = XSDataInputMXThumbnail() xsDataInputMXThumbnail.image = self.getDataInput( ).getDiffractionImage() xsDataInputMXThumbnail.height = XSDataInteger(1024) xsDataInputMXThumbnail.width = XSDataInteger(1024) xsDataInputMXThumbnail.format = self.dataInput.format # Output path strImageNameWithoutExt = os.path.basename( os.path.splitext(strPathToDiffractionImage)[0]) strImageDirname = os.path.dirname(strPathToDiffractionImage) if self.getDataInput().getForcedOutputDirectory(): strForcedOutputDirectory = self.getDataInput( ).getForcedOutputDirectory().getPath().getValue() if not os.access(strForcedOutputDirectory, os.W_OK): self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory) self.setFailure() else: self.strOutputPathWithoutExtension = os.path.join( strForcedOutputDirectory, strImageNameWithoutExt) else: # Try to store in the ESRF pyarch directory strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath( strImageDirname) # Check that output pyarch path exists and is writeable: bIsOk = False if strOutputDirname: if not os.path.exists(strOutputDirname): # Try to create the directory try: os.makedirs(strOutputDirname) bIsOk = True except Exception as e: self.WARNING("Couldn't create the directory %s" % strOutputDirname) elif os.access(strOutputDirname, os.W_OK): bIsOk = True if not bIsOk: self.warning("Cannot write to pyarch directory: %s" % strOutputDirname) strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0o755) strOutputDirname = tempfile.mkdtemp( prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser) os.chmod(strOutputDirname, 0o755) self.warning("Writing thumbnail images to: %s" % strOutputDirname) self.strOutputPathWithoutExtension = os.path.join( strOutputDirname, strImageNameWithoutExt) if self.dataInput.format is not None: self.strSuffix = self.dataInput.format.value.lower() self.strImageFormat = self.dataInput.format.value.upper() self.strOutputPath = os.path.join( self.strOutputPathWithoutExtension + "." + self.strSuffix) xsDataInputMXThumbnail.setOutputPath( XSDataFile(XSDataString(self.strOutputPath))) self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def createPyarchFilePath(_strESRFPath): """ This method translates from an ESRF "visitor" path to a "pyarch" path: /data/visitor/mx415/id14eh1/20100209 -> /data/pyarch/2010/id14eh1/mx415/20100209 """ strPyarchDNAFilePath = None listOfDirectories = _strESRFPath.split(os.sep) if EDUtilsPath.isALBA(): return EDHandlerESRFPyarchv1_0.translateToIspybALBAPath( _strESRFPath) if EDUtilsPath.isEMBL(): if 'p13' in listOfDirectories[0:3] or 'P13' in listOfDirectories[ 0:3]: strPyarchDNAFilePath = os.path.join('/data/ispyb/p13', *listOfDirectories[4:]) else: strPyarchDNAFilePath = os.path.join('/data/ispyb/p14', *listOfDirectories[4:]) return strPyarchDNAFilePath if EDUtilsPath.isMAXIV(): strPyarchDNAFilePath = _strESRFPath.replace( "/data", "/mxn/groups/ispybstorage", 1) return strPyarchDNAFilePath listBeamlines = [ "bm07", "id14eh1", "id14eh2", "id14eh3", "id14eh4", "id23eh1", "id23eh2", "id29", "id30a1", "id30a2", "id30a3", "id30b", "simulator_mxcube" ] # Check that we have at least four levels of directories: if (len(listOfDirectories) > 5): strDataDirectory = listOfDirectories[1] strSecondDirectory = listOfDirectories[2] strThirdDirectory = listOfDirectories[3] strFourthDirectory = listOfDirectories[4] strFifthDirectory = listOfDirectories[5] year = strFifthDirectory[0:4] strProposal = None strBeamline = None if (strDataDirectory == "data") and (strSecondDirectory == "gz"): if strThirdDirectory == "visitor": strProposal = strFourthDirectory strBeamline = strFifthDirectory elif strFourthDirectory == "inhouse": strProposal = strFifthDirectory strBeamline = strThirdDirectory else: raise RuntimeError( "Illegal path for EDHandlerESRFPyarchv1_0.createPyarchFilePath: {0}" .format(_strESRFPath)) listOfRemainingDirectories = listOfDirectories[6:] elif (strDataDirectory == "data") and (strSecondDirectory == "visitor"): strProposal = listOfDirectories[3] strBeamline = listOfDirectories[4] listOfRemainingDirectories = listOfDirectories[5:] elif ((strDataDirectory == "data") and (strSecondDirectory in listBeamlines)): strBeamline = strSecondDirectory strProposal = listOfDirectories[4] listOfRemainingDirectories = listOfDirectories[5:] if (strProposal != None) and (strBeamline != None): strPyarchDNAFilePath = os.path.join(os.sep, "data") strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, "pyarch") strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, year) strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strBeamline) strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strProposal) for strDirectory in listOfRemainingDirectories: strPyarchDNAFilePath = os.path.join( strPyarchDNAFilePath, strDirectory) if (strPyarchDNAFilePath is None): EDVerbose.WARNING( "EDHandlerESRFPyarchv1_0.createPyArchFilePath: path not converted for pyarch: %s " % _strESRFPath) return strPyarchDNAFilePath