def input_from_params(self, data_collection, char_params): edna_input = XSDataInputMXCuBE.parseString(self.edna_default_input) if data_collection.id: edna_input.setDataCollectionId(XSDataInteger(data_collection.id)) # Beam object beam = edna_input.getExperimentalCondition().getBeam() try: transmission = HWR.beamline.transmission.get_value() beam.setTransmission(XSDataDouble(transmission)) except AttributeError: import traceback logging.getLogger("HWR").debug( "EDNACharacterisation. transmission not saved ") logging.getLogger("HWR").debug(traceback.format_exc()) try: wavelength = HWR.beamline.energy.get_wavelength() beam.setWavelength(XSDataWavelength(wavelength)) except AttributeError: pass try: beam.setFlux(XSDataFlux(HWR.beamline.flux.get_value())) except AttributeError: pass try: min_exp_time = self.collect_obj.detector_hwobj.get_exposure_time_limits( )[0] beam.setMinExposureTimePerImage(XSDataTime(min_exp_time)) except AttributeError: pass try: beamsize = self.collect_obj.beam_info_hwobj.get_beam_size() if None not in beamsize: beam.setSize( XSDataSize( x=XSDataLength(float(beamsize[0])), y=XSDataLength(float(beamsize[1])), )) except AttributeError: pass # Optimization parameters diff_plan = edna_input.getDiffractionPlan() aimed_i_sigma = XSDataDouble(char_params.aimed_i_sigma) aimed_completness = XSDataDouble(char_params.aimed_completness) aimed_multiplicity = XSDataDouble(char_params.aimed_multiplicity) aimed_resolution = XSDataDouble(char_params.aimed_resolution) complexity = char_params.strategy_complexity complexity = XSDataString(qme.STRATEGY_COMPLEXITY[complexity]) permitted_phi_start = XSDataAngle(char_params.permitted_phi_start) _range = char_params.permitted_phi_end - char_params.permitted_phi_start rotation_range = XSDataAngle(_range) if char_params.aimed_i_sigma: diff_plan.setAimedIOverSigmaAtHighestResolution(aimed_i_sigma) if char_params.aimed_completness: diff_plan.setAimedCompleteness(aimed_completness) if char_params.use_aimed_multiplicity: diff_plan.setAimedMultiplicity(aimed_multiplicity) if char_params.use_aimed_resolution: diff_plan.setAimedResolution(aimed_resolution) diff_plan.setComplexity(complexity) if char_params.use_permitted_rotation: diff_plan.setUserDefinedRotationStart(permitted_phi_start) diff_plan.setUserDefinedRotationRange(rotation_range) # Vertical crystal dimension sample = edna_input.getSample() sample.getSize().setY(XSDataLength(char_params.max_crystal_vdim)) sample.getSize().setZ(XSDataLength(char_params.min_crystal_vdim)) # Radiation damage model sample.setSusceptibility(XSDataDouble(char_params.rad_suscept)) sample.setChemicalComposition(None) sample.setRadiationDamageModelBeta(XSDataDouble(char_params.beta / 1e6)) sample.setRadiationDamageModelGamma( XSDataDouble(char_params.gamma / 1e6)) diff_plan.setForcedSpaceGroup(XSDataString(char_params.space_group)) # Characterisation type - Routine DC if char_params.use_min_dose: pass if char_params.use_min_time: time = XSDataTime(char_params.min_time) diff_plan.setMaxExposureTimePerDataCollection(time) # Account for radiation damage if char_params.induce_burn: self._modify_strategy_option(diff_plan, "-DamPar") # Characterisation type - SAD if char_params.opt_sad: if char_params.auto_res: diff_plan.setAnomalousData(XSDataBoolean(True)) else: diff_plan.setAnomalousData(XSDataBoolean(False)) self._modify_strategy_option(diff_plan, "-SAD yes") diff_plan.setAimedResolution(XSDataDouble(char_params.sad_res)) else: diff_plan.setAnomalousData(XSDataBoolean(False)) # Data set data_set = XSDataMXCuBEDataSet() acquisition_parameters = data_collection.acquisitions[ 0].acquisition_parameters path_template = data_collection.acquisitions[0].path_template path_str = os.path.join(path_template.directory, path_template.get_image_file_name()) for img_num in range(int(acquisition_parameters.num_images)): image_file = XSDataFile() path = XSDataString() path.setValue(path_str % (img_num + 1)) image_file.setPath(path) data_set.addImageFile(image_file) edna_input.addDataSet(data_set) edna_input.process_directory = path_template.process_directory return edna_input
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_6.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist" % dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist" % inp) time.sleep(1.0) shutil.copyfile(inp, dst) self.addExecutiveSummaryLine( "Got only one frame ... nothing to merge !!!") else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], self.lstInput[1]]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.addExecutiveSummaryLine("WARNING: " + self.strRadiationDamage) self.addExecutiveSummaryLine("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) # outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin( self.__strControlledPluginAutoSub) base = "_".join( os.path.basename( self.__class__.lastSample.path.value).split("_")[:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) self.xsdSubtractedCurve = XSDataFile(XSDataString(sub)) #self.curves.append(xsdSubtractedCurve) self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub( sampleCurve=self.__class__.lastSample, buffers=[ self.__class__.lastBuffer, self.dataInput.mergedCurve ], subtractedCurve=self.xsdSubtractedCurve) self.__edPluginExecAutoSub.connectSUCCESS( self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE( self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() if self.isFailure(): return self.__edPluginSaxsAnalysis = self.loadPlugin( self.__strControlledPluginSaxsAnalysis) self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis( scatterCurve=self.xsdSubtractedCurve, autoRg=self.autoRg, graphFormat=XSDataString("png")) self.__edPluginSaxsAnalysis.connectSUCCESS( self.doSuccessSaxsAnalysis) self.__edPluginSaxsAnalysis.connectFAILURE( self.doFailureSaxsAnalysis) self.__edPluginSaxsAnalysis.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve #self.__class__.lastSample = None #Information neededfor transfer to ISPyB self.forgetLastSample = True else: self.__class__.lastSample = self.dataInput.mergedCurve if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID: self.addExecutiveSummaryLine("Registering to ISPyB") self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged)) self.__class__.dictFrames[self.dataInput.mergedCurve] = { 'averaged': self.lstMerged, 'discarded': self.lstDiscarded } self.__edPluginSaxsISPyB = self.loadPlugin( self.__strControlledPluginSaxsISPyB) if len(self.lstInput) > 1: frameAverage = XSDataInteger(len(self.lstInput)) frameMerged = XSDataInteger(len(self.lstMerged)) else: frameMerged = frameAverage = XSDataInteger(1) self.curves = [XSDataFile(i.path) for i in self.lstInput] self.discardedCurves = [ XSDataFile(i.path) for i in self.lstDiscarded ] self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged] averageFilePath = None if self.strMergedFile is not None: averageFilePath = XSDataFile(XSDataString(self.strMergedFile)) self.sampleFrames = self.getAveragedFrameByFilename( self.__class__.lastSample) lastSample = None if self.__class__.lastSample is not None: lastSample = self.__class__.lastSample subtractedCurve = None if self.xsdSubtractedCurve is not None: subtractedCurve = self.xsdSubtractedCurve xsdin = XSDataInputBioSaxsISPyBv1_0( sample=self.dataInput.sample, autoRg=self.autoRg, gnom=self.gnom, volume=self.volume, frameAverage=frameAverage, frameMerged=frameMerged, curves=self.curves, discardedFrames=self.discardedCurves, averagedFrames=self.mergedCurves, averageFilePath=averageFilePath, bufferFrames=self.bufferFrames, sampleFrames=self.sampleFrames, bestBuffer=self.xsBestBuffer, averageSample=lastSample, scatterPlot=self.xsScatterPlot, guinierPlot=self.xsGuinierPlot, kratkyPlot=self.xsKratkyPlot, densityPlot=self.xsDensityPlot, subtractedFilePath=subtractedCurve # destination=self.dataInput.sample.ispybDestination #duplicate, already in sample ) self.__edPluginSaxsISPyB.dataInput = xsdin self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB) self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB) self.__edPluginSaxsISPyB.execute() if self.forgetLastSample: #Also redefine dictionary to contain the buffer just processed? self.__class__.lastSample = None
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection( ) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace( "%05d", "####") else: template = ispybDataCollection.fileTemplate.replace( "%04d", "####") if self.dataInput.startFrame is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.startFrame.value if self.dataInput.endFrame is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.endFrame.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoEnd) # else: # directory = self.dataInput.dirN.value # template = self.dataInput.templateN.value # imageNoStart = self.dataInput.fromN.value # imageNoEnd = self.dataInput.toN.value # fileTemplate = template.replace("####", "%04d") # pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) # pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace( 'PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0o755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isEMBL(): fWaitFileTimeout = 60 else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile( XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % ( fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() # Prepare input to execution plugin xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS() xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True) xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS() xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False) xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell if imageNoStart is not None: xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger( imageNoStart) if imageNoEnd is not None: xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd) if isH5: masterFilePath = os.path.join( directory, self.eiger_template_to_master(template)) xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(masterFilePath))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(masterFilePath))) else: xsDataInputXia2DIALSAnom.addImage( XSDataFile(XSDataString(pathToStartImage))) if self.doAnomAndNonanom: xsDataInputXia2DIALSNoanom.addImage( XSDataFile(XSDataString(pathToStartImage))) self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doAnomAndNonanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom self.edPluginExecXia2DIALSAnom.execute() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom self.edPluginExecXia2DIALSNoanom.execute() self.edPluginExecXia2DIALSAnom.synchronize() if self.doAnomAndNonanom: self.edPluginExecXia2DIALSNoanom.synchronize() self.timeEnd = time.localtime() # Upload to ISPyB self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSAnom, True, proposal, self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doAnomAndNonanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB( self.edPluginExecXia2DIALSNoanom, False, proposal, self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("*** EDPluginExecReadImageHeaderADSCv10.process") xsDataInputReadImageHeader = self.getDataInput() xsDataFile = xsDataInputReadImageHeader.getImage() strPath = xsDataFile.getPath().getValue() strAbsolutePath = os.path.abspath(strPath) dictHeader = self.readHeaderADSC(strPath) if (dictHeader is None): strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : error when reading header from %s" % strAbsolutePath self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() else: xsDataExperimentalCondition = XSDataExperimentalCondition() xsDataDetector = XSDataDetector() xsDataDetector.setBeamPositionX( XSDataLength(float(dictHeader["BEAM_CENTER_X"]))) xsDataDetector.setBeamPositionY( XSDataLength(float(dictHeader["BEAM_CENTER_Y"]))) xsDataDetector.setDistance( XSDataLength(float(dictHeader["DISTANCE"]))) fPixelSize = float(dictHeader["PIXEL_SIZE"]) xsDataDetector.setPixelSizeX(XSDataLength(fPixelSize)) xsDataDetector.setPixelSizeY(XSDataLength(fPixelSize)) if "TWOTHETA" in dictHeader.keys(): xsDataDetector.setTwoTheta( XSDataAngle(float(dictHeader["TWOTHETA"]))) xsDataDetector.setNumberBytesInHeader( XSDataInteger(float(dictHeader["HEADER_BYTES"]))) xsDataDetector.setSerialNumber( XSDataString(dictHeader["DETECTOR_SN"])) xsDataDetector.setNumberPixelX( XSDataInteger(int(dictHeader["SIZE1"]))) xsDataDetector.setNumberPixelY( XSDataInteger(int(dictHeader["SIZE2"]))) xsDataDetector.setBin(XSDataString(dictHeader["BIN"])) xsDataDetector.setDataType(XSDataString(dictHeader["TYPE"])) xsDataDetector.setByteOrder(XSDataString(dictHeader["BYTE_ORDER"])) if "CCD_IMAGE_SATURATION" in dictHeader.keys(): xsDataDetector.setImageSaturation( XSDataInteger(int(dictHeader["CCD_IMAGE_SATURATION"]))) # Determine type of detector... iNoPixelsX = xsDataDetector.getNumberPixelX().getValue() iNoPixelsY = xsDataDetector.getNumberPixelY().getValue() if (iNoPixelsX == 2304 and iNoPixelsY == 2304): xsDataDetector.setName(XSDataString("ADSC Q4")) xsDataDetector.setType(XSDataString("q4")) elif (iNoPixelsX == 1152 and iNoPixelsY == 1152): xsDataDetector.setName(XSDataString("ADSC Q4 bin 2x2")) xsDataDetector.setType(XSDataString("q4-2x")) elif (iNoPixelsX == 4096 and iNoPixelsY == 4096): xsDataDetector.setName(XSDataString("ADSC Q210")) xsDataDetector.setType(XSDataString("q210")) elif (iNoPixelsX == 2048 and iNoPixelsY == 2048): xsDataDetector.setName(XSDataString("ADSC Q210 bin 2x2")) xsDataDetector.setType(XSDataString("q210-2x")) elif (iNoPixelsX == 6144 and iNoPixelsY == 6144): xsDataDetector.setName(XSDataString("ADSC Q315")) xsDataDetector.setType(XSDataString("q315")) elif (iNoPixelsX == 3072 and iNoPixelsY == 3072): xsDataDetector.setName(XSDataString("ADSC Q315 bin 2x2")) xsDataDetector.setType(XSDataString("q315-2x")) else: strErrorMessage = EDMessage.ERROR_DATA_HANDLER_02 % ( "EDPluginExecReadImageHeaderADSCv10.process", "Unknown detector type") self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) xsDataExperimentalCondition.setDetector(xsDataDetector) # Beam object xsDataBeam = XSDataBeam() xsDataBeam.setWavelength( XSDataWavelength(float(dictHeader["WAVELENGTH"]))) xsDataBeam.setExposureTime(XSDataTime(float(dictHeader["TIME"]))) xsDataExperimentalCondition.setBeam(xsDataBeam) # Goniostat object xsDataGoniostat = XSDataGoniostat() fRotationAxisStart = float(dictHeader["OSC_START"]) fOscillationWidth = float(dictHeader["OSC_RANGE"]) xsDataGoniostat.setRotationAxisStart( XSDataAngle(fRotationAxisStart)) xsDataGoniostat.setRotationAxisEnd( XSDataAngle(fRotationAxisStart + fOscillationWidth)) xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth)) strRotationAxis = None if ("AXIS" in dictHeader.keys()): strRotationAxis = dictHeader["AXIS"] elif ("OSC_AXIS" in dictHeader.keys()): strRotationAxis = dictHeader["OSC_AXIS"] else: strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : Neither AXIS nor OSC_AXIS header item found." self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() xsDataGoniostat.setRotationAxis(XSDataString(strRotationAxis)) xsDataExperimentalCondition.setGoniostat(xsDataGoniostat) # Create the image object xsDataImage = XSDataImage() xsDataImage.setPath(XSDataString(strAbsolutePath)) xsDataImage.setDate(XSDataString(dictHeader["DATE"])) strFileName = os.path.basename(strPath) iImageNumber = EDUtilsImage.getImageNumber(strFileName) xsDataImage.setNumber(XSDataInteger(iImageNumber)) xsDataSubWedge = XSDataSubWedge() xsDataSubWedge.setExperimentalCondition( xsDataExperimentalCondition) xsDataSubWedge.addImage(xsDataImage) self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader() self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen( "Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath( strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen( "ID30a3 Eiger data, waiting for master and data files..." ) listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen( "Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile( listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger( firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger( lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile( XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin( "EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:06d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join( directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace( ".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber( image.path.value) oldPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:04d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) )) strPathToImage = os.path.join( directory, lastCbfFile) self.screen( "Image has been converted to CBF file: {0}" .format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen( "Still waiting for converting to CBF file: {0}" .format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger( EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage( xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append( (xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_{0}".format( os.path.splitext( os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append( (edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen( "No dozor results! Re-executing Dozor for images {0} to {1}" .format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format( os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile( image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists( imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt( imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString( base64.b64encode( numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput( ).parseString( edPluginControlDozor.dataOutput. inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[ -5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader( ) xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path ) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin( self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing( ) xsDataInputLabelitIndexing.image.append( XSDataImage( xsDataResultControlImageQualityIndicator. image.path)) edPluginLabelitIndexing.setDataInput( xsDataInputLabelitIndexing) self.listPluginLabelit.append([ edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator ]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure( ) and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput( ) xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult( xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def buildChildren(self, child_, nodeName_): if child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'experimentalCondition': obj_ = XSDataExperimentalCondition() obj_.build(child_) self.setExperimentalCondition(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'diffractionPlan': obj_ = XSDataDiffractionPlan() obj_.build(child_) self.setDiffractionPlan(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'sample': obj_ = XSDataSampleCrystalMM() obj_.build(child_) self.setSample(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'imagePath': obj_ = XSDataImage() obj_.build(child_) self.imagePath.append(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'flux': obj_ = XSDataFloat() obj_.build(child_) self.setFlux(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'minExposureTimePerImage': obj_ = XSDataTime() obj_.build(child_) self.setMinExposureTimePerImage(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamSize': obj_ = XSDataLength() obj_.build(child_) self.setBeamSize(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamSizeX': obj_ = XSDataLength() obj_.build(child_) self.setBeamSizeX(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamSizeY': obj_ = XSDataLength() obj_.build(child_) self.setBeamSizeY(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'templateMode': obj_ = XSDataBoolean() obj_.build(child_) self.setTemplateMode(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'generatedTemplateFile': obj_ = XSDataFile() obj_.build(child_) self.setGeneratedTemplateFile(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'resultsFilePath': obj_ = XSDataFile() obj_.build(child_) self.setResultsFilePath(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamPosX': obj_ = XSDataFloat() obj_.build(child_) self.setBeamPosX(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamPosY': obj_ = XSDataFloat() obj_.build(child_) self.setBeamPosY(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'wavelength': obj_ = XSDataWavelength() obj_.build(child_) self.setWavelength(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'transmission': obj_ = XSDataDouble() obj_.build(child_) self.setTransmission(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'dataCollectionId': obj_ = XSDataInteger() obj_.build(child_) self.setDataCollectionId(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'shortComments': obj_ = XSDataString() obj_.build(child_) self.setShortComments(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'comments': obj_ = XSDataString() obj_.build(child_) self.setComments(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'inputCharacterisation': obj_ = XSDataInputCharacterisation() obj_.build(child_) self.setInputCharacterisation(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'token': obj_ = XSDataString() obj_.build(child_) self.setToken(obj_)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlAutoPROCv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection() identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory if EDUtilsPath.isEMBL(): template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5) elif EDUtilsPath.isMAXIV(): template = ispybDataCollection.fileTemplate else: template = ispybDataCollection.fileTemplate.replace("%04d", "####") if self.dataInput.fromN is None: imageNoStart = ispybDataCollection.startImageNumber else: imageNoStart = self.dataInput.fromN.value if self.dataInput.toN is None: imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1 else: imageNoEnd = self.dataInput.toN.value # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.path.value template = self.dataInput.templateN.value imageNoStart = self.dataInput.fromN.value imageNoEnd = self.dataInput.toN.value if EDUtilsPath.isEMBL(): fileTemplate = template.replace("#####", "%05d") else: fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory if EDUtilsPath.isALBA(): _processDirectory = "_".join(pathToStartImage.split('_')[:-1]) from datetime import datetime _id = datetime.now().strftime('%Y%m%d_%H%M%S') self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id) else: self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline, "autoPROC", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) if self.pyarchDirectory is not None: self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if not os.path.exists(self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # The resultsDirectory is not used at ALBA (only pyarchDirectory) if EDUtilsPath.isALBA(): self.resultsDirectory = None # Determine pyarch prefix if EDUtilsPath.isALBA(): listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]), listPrefix[-2]) else: listPrefix = template.split("_") self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id30b"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 if EDUtilsPath.isMAXIV(): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join(directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV(): fWaitFileTimeout = 60 # s else: fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() self.timeStart = time.localtime() if self.dataInput.dataCollectionId is not None: # Set ISPyB to running if self.doAnom: self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=True, timeStart=self.timeStart) self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=True, timeStart=self.timeStart) if self.doNoanom: self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgram, isAnom=False, timeStart=self.timeStart) self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingProgramStaraniso, isAnom=False, timeStart=self.timeStart) # Prepare input to execution plugin if self.doAnom: xsDataInputAutoPROCAnom = XSDataInputAutoPROC() xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True) xsDataInputAutoPROCAnom.symm = self.dataInput.symm xsDataInputAutoPROCAnom.cell = self.dataInput.cell xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit if self.doNoanom: xsDataInputAutoPROCNoanom = XSDataInputAutoPROC() xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False) xsDataInputAutoPROCNoanom.symm = self.dataInput.symm xsDataInputAutoPROCNoanom.cell = self.dataInput.cell xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier() xsDataAutoPROCIdentifier.idN = XSDataString(identifier) xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory)) xsDataAutoPROCIdentifier.templateN = XSDataString(template) xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart) xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd) if self.doAnom: xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier) if self.doNoanom: xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy()) if isH5: masterFilePath = os.path.join(directory, self.eiger_template_to_master(template)) if self.doAnom: xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath)) if self.doNoanom: xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath)) timeStart = time.localtime() if self.doAnom: self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom self.edPluginExecAutoPROCAnom.execute() if self.doNoanom: self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom self.edPluginExecAutoPROCNoanom.execute() if self.doAnom: self.edPluginExecAutoPROCAnom.synchronize() if self.doNoanom: self.edPluginExecAutoPROCNoanom.synchronize() timeEnd = time.localtime() # Upload to ISPyB if self.doAnom: self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd) if self.doNoanom: self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd) self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) listPlugin.append(edPluginPluginExecImageQualityIndicator) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for edPluginPluginExecImageQualityIndicator in listPlugin: edPluginPluginExecImageQualityIndicator.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.bDoISPyBUpload: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edObject=None): EDPluginExec.process(self) EDVerbose.DEBUG("EDPluginExecReadImageHeaderMARCCDv10.process") xsDataInputReadImageHeader = self.getDataInput() xsDataFile = xsDataInputReadImageHeader.getImage() strPath = xsDataFile.getPath().getValue() dictMARCCDHeader = self.readHeaderMarccd(strPath) if (dictMARCCDHeader is None): strErrorMessage = "EDPluginExecReadImageHeaderMARCCDv10.process : Cannot read header : %s" % strPath EDVerbose.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() else: xsDataExperimentalCondition = XSDataExperimentalCondition() xsDataDetector = XSDataDetector() iNoPixelsX = int(dictMARCCDHeader["nslow"]) iNoPixelsY = int(dictMARCCDHeader["nfast"]) xsDataDetector.setNumberPixelX(XSDataInteger(iNoPixelsX)) xsDataDetector.setNumberPixelY(XSDataInteger(iNoPixelsY)) fPixelSizeX = float(dictMARCCDHeader["pixelsize_x"]) / 1000.0 xsDataDetector.setPixelSizeX(XSDataLength(fPixelSizeX)) fPixelSizeY = float(dictMARCCDHeader["pixelsize_y"]) / 1000.0 xsDataDetector.setPixelSizeY(XSDataLength(fPixelSizeY)) fBeamPositionX = float(dictMARCCDHeader["beam_x"]) / 1000.0 fBeamPositionY = float(dictMARCCDHeader["beam_y"]) / 1000.0 # Fix for bug 397 - check if the beam position is close to the centre of the image fTwoTheta = float(dictMARCCDHeader["end_twotheta"]) / 1000.0 xsDataDetector.setTwoTheta(XSDataAngle(fTwoTheta)) if (abs(fTwoTheta) < 0.1): if (abs(fBeamPositionX / (fPixelSizeX / 1000.0) - iNoPixelsX / 2.0) > (2 * iNoPixelsX)): fBeamPositionX = fBeamPositionX * fPixelSizeX / 1000.0 fBeamPositionY = fBeamPositionY * fPixelSizeY / 1000.0 xsDataDetector.setBeamPositionX(XSDataLength(fBeamPositionX)) xsDataDetector.setBeamPositionY(XSDataLength(fBeamPositionY)) fDistance = float(dictMARCCDHeader["xtal_to_detector"]) / 1000.0 if (abs(fDistance) < 0.1): fDistanceStart = float( dictMARCCDHeader["start_xtal_to_detector"]) / 1000.0 fDistanceEnd = float( dictMARCCDHeader["end_xtal_to_detector"]) / 1000.0 if (abs(fDistanceStart - fDistanceEnd) < 0.1): fDistance = fDistanceStart else: # Somethings very wrong with the distances... strErrorMessage = "EDPluginExecReadImageHeaderMARCCDv10.process : Inconsistency in MAR CCD image header: start_xtal_to_detector = %d, end_xtal_to_detector = %d" % \ (fDistanceStart, fDistanceEnd) EDVerbose.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() xsDataDetector.setDistance(XSDataLength(fDistance)) xsDataDetector.setNumberBytesInHeader( XSDataInteger(float(dictMARCCDHeader["header_size"]))) #xsDataDetector.setSerialNumber( XSDataInteger( dictMARCCDHeader[ "DETECTOR_SN" ] ) ) ) #xsDataDetector.setBin( XSDataString( dictMARCCDHeader[ "BIN" ] ) ) ) #xsDataDetector.setDataType( XSDataString( dictMARCCDHeader[ "TYPE" ] ) ) ) #xsDataDetector.setByteOrder( XSDataString( dictMARCCDHeader[ "BYTE_ORDER" ] ) ) ) xsDataDetector.setImageSaturation( XSDataInteger(int(dictMARCCDHeader["saturation_level"]))) # Determine type of detector... if (iNoPixelsX == 2048 and iNoPixelsY == 2048): xsDataDetector.setName(XSDataString("MAR CCD 165")) xsDataDetector.setType(XSDataString("mar165")) elif (iNoPixelsX == 3072 and iNoPixelsY == 3072): xsDataDetector.setName(XSDataString("MAR CCD 225")) xsDataDetector.setType(XSDataString("mar225")) elif (iNoPixelsX == 4096 and iNoPixelsY == 4096): xsDataDetector.setName(XSDataString("MAR CCD 325")) xsDataDetector.setType(XSDataString("mar325")) else: strErrorMessage = EDMessage.ERROR_DATA_HANDLER_02 % ( "EDPluginExecReadImageHeaderMARCCDv10.process", "Unknown detector type") EDVerbose.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage xsDataExperimentalCondition.setDetector(xsDataDetector) # Beam object xsDataBeam = XSDataBeam() xsDataBeam.setWavelength( XSDataWavelength( float(dictMARCCDHeader["source_wavelength"]) / 100000.0)) xsDataBeam.setExposureTime( XSDataTime(float(dictMARCCDHeader["exposure_time"]) / 1000.0)) xsDataExperimentalCondition.setBeam(xsDataBeam) # Goniostat object xsDataGoniostat = XSDataGoniostat() fRotationAxisStart = float(dictMARCCDHeader["start_phi"]) / 1000.0 fOscillationWidth = float( dictMARCCDHeader["rotation_range"]) / 1000.0 xsDataGoniostat.setRotationAxisStart( XSDataAngle(fRotationAxisStart)) xsDataGoniostat.setRotationAxisEnd( XSDataAngle(fRotationAxisStart + fOscillationWidth)) xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth)) xsDataExperimentalCondition.setGoniostat(xsDataGoniostat) # Create the image object xsDataImage = XSDataImage() xsDataImage.setPath(XSDataString(strPath)) strTimeStamp = dictMARCCDHeader["acquire_timestamp"] xsDataImage.setDate(XSDataString(strTimeStamp)) iImageNumber = EDUtilsImage.getImageNumber(strPath) xsDataImage.setNumber(XSDataInteger(iImageNumber)) xsDataSubWedge = XSDataSubWedge() xsDataSubWedge.setExperimentalCondition( xsDataExperimentalCondition) xsDataSubWedge.addImage(xsDataImage) self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader() self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
EDVerbose.screen("Usage: id29_create_thumbnail image_directory_path image1 [image2]" ) sys.exit(1) EDVerbose.screen("Arguments: %r" % sys.argv) # Remove duplicates listImageName = list(set(sys.argv[2:])) # Strip off prefix listPrefix = listImageName[0].split("_") strPrefix = "{0}_{1}".format(listPrefix[0], listPrefix[1]) # Check if a temp working directory should be created if "CREATE_THUMBNAIL_WORKING_DIR" in os.environ: strPathToTempDir = tempfile.mkdtemp(prefix="thumbnail-{0}-".format(strPrefix), dir=os.environ["CREATE_THUMBNAIL_WORKING_DIR"]) else: strTmpUser = os.path.join("/tmp", os.environ["USER"]) if not os.path.exists(strTmpUser): os.mkdir(strTmpUser, 0o755) strPathToTempDir = tempfile.mkdtemp(prefix="thumbnail-{0}-".format(strPrefix), dir=strTmpUser) os.chmod(strPathToTempDir, 0o755) os.chdir(strPathToTempDir) EDVerbose.setLogFileName(os.path.join(strPathToTempDir, "id29_create_thumbnail.log")) strImageDirectory = sys.argv[1] for strImageName in listImageName: xsDataInputPyarchThumbnailGenerator = XSDataInputPyarchThumbnailGenerator() xsDataInputPyarchThumbnailGenerator.setWaitForFileTimeOut(XSDataTime(1000)) strImagePath = os.path.join(strImageDirectory, strImageName) xsDataInputPyarchThumbnailGenerator.setDiffractionImage(XSDataFile(XSDataString(strImagePath))) EDVerbose.screen("XML input for EDPluginControlPyarchThumbnailGeneratorv1_0: %s" % xsDataInputPyarchThumbnailGenerator.marshal()) edPluginControlPyarchThumbnailGeneratorv1_0 = EDPluginControlPyarchThumbnailGeneratorv1_0() edPluginControlPyarchThumbnailGeneratorv1_0.setDataInput(xsDataInputPyarchThumbnailGenerator) edPluginControlPyarchThumbnailGeneratorv1_0.execute()
def preProcess(self, _edObject = None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict(creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format(self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath(os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath(os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def testSetDataModelInput(self): # Crystal from XSDataRaddosev10 import XSDataRaddoseInput xsDataRaddoseInput = XSDataRaddoseInput() from XSDataCommon import XSDataString from XSDataCommon import XSDataFloat from XSDataCommon import XSDataAngle from XSDataCommon import XSDataLength from XSDataCommon import XSDataSize from XSDataCommon import XSDataInteger from XSDataRaddosev10 import XSDataCell from XSDataRaddosev10 import XSDataAtom from XSDataRaddosev10 import XSDataAtomicComposition xsDataAtomSulfur = XSDataAtom() xsDataAtomSulfur.setNumberOf(XSDataFloat(4)) xsDataAtomSulfur.setSymbol(XSDataString("S")) xsDataAtomSelenium = XSDataAtom() xsDataAtomSelenium.setNumberOf(XSDataFloat(4)) xsDataAtomSelenium.setSymbol(XSDataString("Se")) xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtomicComposition.addAtom(xsDataAtomSulfur) xsDataAtomicComposition.addAtom(xsDataAtomSelenium) xsDataRaddoseInput.setCrystalPATM(xsDataAtomicComposition) xsDataRaddoseInput.setCrystalNRES(XSDataInteger(295)) xsDataRaddoseInput.setCrystalNMON(XSDataInteger(8)) xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0), angle_beta=XSDataAngle(90.0), angle_gamma=XSDataAngle(90.0), length_a=XSDataLength(78.9), length_b=XSDataLength(95.162), length_c=XSDataLength(104.087)) xsDataSizeCrystal = XSDataSize(XSDataLength(0.1), XSDataLength(0.1), XSDataLength(0.1)) xsDataRaddoseInput.setCrystalCell(xsDataCell) xsDataRaddoseInput.setCrystalSize(xsDataSizeCrystal) # Beam from XSDataCommon import XSDataFlux from XSDataCommon import XSDataWavelength from XSDataCommon import XSDataTime xsDataSize = XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1)) xsDataRaddoseInput.setBeamSize(xsDataSize) xsDataRaddoseInput.setBeamFlux(XSDataFlux(1e+12)) xsDataRaddoseInput.setBeamWavelength(XSDataWavelength(2.41)) xsDataRaddoseInput.setBeamExposureTime(XSDataTime(1)) xsDataRaddoseInput.setNumberOfImages(XSDataInteger(1)) xsDataRaddoseInput.exportToFile(self.strObtainedInputFile) strExpectedInput = self.readAndParseFile(self.strReferenceInputFile) strObtainedInput = self.readAndParseFile(self.strObtainedInputFile) xsDataRaddoseInputExpected = XSDataRaddoseInput.parseString( strExpectedInput) xsDataRaddoseInputObtained = XSDataRaddoseInput.parseString( strObtainedInput) EDAssert.equal(xsDataRaddoseInputExpected.marshal(), xsDataRaddoseInputObtained.marshal())
def postProcess(self, _edObject=None): EDPluginExecProcessScript.postProcess(self) strRaddoseError = self.readProcessErrorLogFile() if ((strRaddoseError is not None) and (strRaddoseError != "")): errorMessage = EDMessage.ERROR_EXECUTION_03 % ( 'EDPluginRaddosev10.postProcess', 'EDPluginRaddosev10', strRaddoseError) self.error(errorMessage) self.addErrorMessage(errorMessage) raise RuntimeError(errorMessage) resultDict = self.analyseScriptLogFileName( os.path.join(self.getWorkingDirectory(), self.getScriptLogFileName())) strSolvent = self.__dictResults[EDPluginRaddosev10.__strSOLVENT] # ugly workaround while waiting for RADDOSE XML output file strAbsorbedDose = None for strAbsorbedDoseKeyword in EDPluginRaddosev10.__listABSORBED_DOSE: strAbsorbedDose = self.__dictResults[strAbsorbedDoseKeyword] if (strAbsorbedDose is not None): break if (strSolvent is None): errorMessage = EDMessage.ERROR_EXECUTION_03 % ( 'EDPluginRaddosev10.postProcess', "Raddose", "No Result for Keyword [" + EDPluginRaddosev10.__strSOLVENT + "] see: " + self.getScriptLogFileName()) self.error(errorMessage) self.addErrorMessage(errorMessage) raise RuntimeError(errorMessage) if (strAbsorbedDose is None): errorMessage = EDMessage.ERROR_EXECUTION_03 % ( 'EDPluginRaddosev10.postProcess', "Raddose", "No Result for Keyword [" + strAbsorbedDoseKeyword + "] see: " + self.getScriptLogFileName()) self.error(errorMessage) self.addErrorMessage(errorMessage) raise RuntimeError(errorMessage) self.__fSolvent = float(strSolvent) if (self.__fSolvent < EDPluginRaddosev10.__iMIN_SOLVENT_PERCENTAGE or self.__fSolvent > EDPluginRaddosev10.__iMAX_SOLVENT_PERCENTAGE): warningMessage = "Inconsistent solvent percentage value: %.1f" % self.__fSolvent self.warning(warningMessage) self.addWarningMessage(warningMessage) iNumberOfImages = self.getDataInput().getNumberOfImages().getValue() fExposureTimePerImageInSec = self.getDataInput().getBeamExposureTime( ).getValue() fTotalExposureTime = iNumberOfImages * fExposureTimePerImageInSec fAbsorbedDoseSpeed = float(strAbsorbedDose) / fTotalExposureTime xsDataAbsorbedDoseRate = XSDataAbsorbedDoseRate(fAbsorbedDoseSpeed) xsDataAbsorbedDose = XSDataDouble(float(strAbsorbedDose)) self.__fTimeToReachHendersonLimit = EDPluginRaddosev10.__fHENDERSON_LIMIT / fAbsorbedDoseSpeed xsDataRaddosev10Output = XSDataRaddoseOutput() xsDataRaddosev10Output.setAbsorbedDose(xsDataAbsorbedDose) xsDataRaddosev10Output.setAbsorbedDoseRate(xsDataAbsorbedDoseRate) xsDataRaddosev10Output.setTimeToReachHendersonLimit( XSDataTime(self.__fTimeToReachHendersonLimit)) xsDataFilePathToLog = XSDataFile( XSDataString( os.path.join(self.getWorkingDirectory(), self.getScriptLogFileName()))) xsDataRaddosev10Output.setPathToLogFile(xsDataFilePathToLog) self.setDataOutput(xsDataRaddosev10Output)
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict( creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format( self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join( os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath( os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath( os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath( os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format( data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def buildChildren(self, child_, nodeName_): if child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamExposureTime': obj_ = XSDataTime() obj_.build(child_) self.setBeamExposureTime(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamFlux': obj_ = XSDataFlux() obj_.build(child_) self.setBeamFlux(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamSize': obj_ = XSDataSize() obj_.build(child_) self.setBeamSize(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamWavelength': obj_ = XSDataWavelength() obj_.build(child_) self.setBeamWavelength(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalCell': obj_ = XSDataCell() obj_.build(child_) self.setCrystalCell(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNDNA': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNDNA(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNMON': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNMON(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNRES': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNRES(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNRNA': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNRNA(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalPATM': obj_ = XSDataAtomicComposition() obj_.build(child_) self.setCrystalPATM(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalSATM': obj_ = XSDataAtomicComposition() obj_.build(child_) self.setCrystalSATM(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalSize': obj_ = XSDataSize() obj_.build(child_) self.setCrystalSize(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'numberOfImages': obj_ = XSDataInteger() obj_.build(child_) self.setNumberOfImages(obj_) XSDataInput.buildChildren(self, child_, nodeName_)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG('EDPluginControlXDSAPPv1_0.process starting') directory = None template = None imageNoStart = None imageNoEnd = None pathToStartImage = None pathToEndImage = None userName = os.environ["USER"] beamline = "unknown" proposal = "unknown" if self.dataInput.startImageNumber is not None: imageNoStart = self.dataInput.startImageNumber.value if self.dataInput.endImageNumber is not None: imageNoEnd = self.dataInput.endImageNumber.value # If we have a data collection id, use it if self.dataInput.dataCollectionId is not None: # Recover the data collection from ISPyB xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection( ) identifier = str(self.dataInput.dataCollectionId.value) xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection self.edPluginRetrieveDataCollection.executeSynchronous() ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection directory = ispybDataCollection.imageDirectory template = ispybDataCollection.fileTemplate.replace("%04d", "####") if imageNoStart is None: imageNoStart = ispybDataCollection.startImageNumber if imageNoEnd is None: imageNoEnd = ispybDataCollection.startImageNumber + \ ispybDataCollection.numberOfImages - 1 # # DEBUG we set the end image to 20 in order to speed up things # self.warning("End image set to 20 (was {0})".format(imageNoEnd)) # imageNoEnd = 20 pathToStartImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoStart) pathToEndImage = os.path.join( directory, ispybDataCollection.fileTemplate % imageNoEnd) else: identifier = str(int(time.time())) directory = self.dataInput.dirN.value template = self.dataInput.templateN.value fileTemplate = template.replace("####", "%04d") pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart) pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd) # Try to get proposal from path if EDUtilsPath.isESRF(): listDirectory = directory.split(os.sep) try: if listDirectory[1] == "data": if listDirectory[2] == "visitor": beamline = listDirectory[4] proposal = listDirectory[3] else: beamline = listDirectory[2] proposal = listDirectory[4] except: beamline = "unknown" proposal = userName if imageNoEnd - imageNoStart < 8: error_message = "There are fewer than 8 images, aborting" self.addErrorMessage(error_message) self.ERROR(error_message) self.setFailure() return # Process directory if self.dataInput.processDirectory is not None: processDirectory = self.dataInput.processDirectory.path.value else: processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA") # Make results directory self.resultsDirectory = os.path.join(processDirectory, "results") if not os.path.exists(self.resultsDirectory): os.makedirs(self.resultsDirectory, 0o755) # Create path to pyarch if self.dataInput.reprocess is not None and self.dataInput.reprocess.value: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath( beamline, "XDSAPP", self.dataInput.dataCollectionId.value) else: self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath( self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace( 'PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists( self.pyarchDirectory): try: os.makedirs(self.pyarchDirectory, 0o755) except: self.pyarchDirectory = None # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) isH5 = False if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]): minSizeFirst = 6000000 minSizeLast = 6000000 elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]): minSizeFirst = 2000000 minSizeLast = 2000000 elif any(beamline in pathToStartImage for beamline in ["id30a3"]): minSizeFirst = 100000 minSizeLast = 100000 pathToStartImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoStart)) pathToEndImage = os.path.join( directory, self.eiger_template_to_image(template, imageNoEnd)) isH5 = True else: minSizeFirst = 1000000 minSizeLast = 1000000 fWaitFileTimeout = 3600 # s xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(pathToStartImage)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst) self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.edPluginWaitFileFirst.executeSynchronous() if self.edPluginWaitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, pathToStartImage) self.addWarningMessage(strWarningMessage) self.WARNING(strWarningMessage) xsDataInputMXWaitFileLast = XSDataInputMXWaitFile() xsDataInputMXWaitFileLast.file = XSDataFile( XSDataString(pathToEndImage)) xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout) self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast) self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast self.edPluginWaitFileLast.executeSynchronous() if self.edPluginWaitFileLast.dataOutput.timedOut.value: strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % ( fWaitFileTimeout, pathToEndImage) self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() if self.dataInput.hdf5ToCbfDirectory is not None: dir = self.dataInput.hdf5ToCbfDirectory.path.value pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0] self.timeStart = time.localtime() # Prepare input to execution plugin if self.doAnom: xsDataInputXDSAPPAnom = XSDataInputXDSAPP() xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True) xsDataInputXDSAPPAnom.image = XSDataFile( XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPAnom.spacegroup = XSDataString( self.xdsAppSpacegroup) self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom self.edPluginExecXDSAPPAnom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=True, timeStart=self.timeStart) if self.doNoanom: xsDataInputXDSAPPNoanom = XSDataInputXDSAPP() xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False) xsDataInputXDSAPPNoanom.image = XSDataFile( XSDataString(pathToStartImage)) if self.xdsAppSpacegroup is not None: xsDataInputXDSAPPNoanom.spacegroup = XSDataString( self.xdsAppSpacegroup) self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom self.edPluginExecXDSAPPNoanom.execute() if self.dataInput.dataCollectionId is not None: # Set ISPyB to started self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \ EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value, processingPrograms=self.processingPrograms, isAnom=False, timeStart=self.timeStart) if self.doAnom: self.edPluginExecXDSAPPAnom.synchronize() xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput # Run XSCALE even if XSCALE.LP is present strPathXscaleLpAnom = self.runXscale( self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True) if self.doNoanom: self.edPluginExecXDSAPPNoanom.synchronize() xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput strPathXscaleLpNoanom = self.runXscale( self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True) self.timeEnd = time.localtime() # Upload to ISPyB if self.dataInput.dataCollectionId is not None: # Check if we should use XDS_ASCII_to_XML.pl if self.doAnom: if self.useXdsAsciiToXml: # Only for anom runs self.runXdsAsciiToXml( xsDataResultXDSAPPAnom, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) else: self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB( xsDataResultXDSAPPAnom, processDirectory, template, strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom) if self.hasUploadedAnomResultsToISPyB: self.screen("Anom results uploaded to ISPyB") else: self.ERROR("Could not upload anom results to ISPyB!") if self.doNoanom: self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB( xsDataResultXDSAPPNoanom, processDirectory, template, strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd, self.dataInput.dataCollectionId.value, self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom) if self.hasUploadedNoanomResultsToISPyB: self.screen("Noanom results uploaded to ISPyB") else: self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlDozorv1_1.process") msg = "EDPluginControlDozorXmlRpcv1_0 started..." self.sendMessageToMXCuBE("Processing started...", "info") xsDataResultControlDozor = XSDataResultControlDozor() firstFileName = self.dataInput.template.value % ( self.dataInput.first_run_number.value, self.dataInput.first_image_number.value) fWaitFileTimeout = 180 #sec self.waitFileFirst = self.loadPlugin("EDPluginMXWaitFilev1_1", "MXWaitFileFirst") xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile() xsDataInputMXWaitFileFirst.file = XSDataFile( XSDataString(firstFileName)) xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout) self.waitFileFirst.size = XSDataInteger(1000000) self.waitFileFirst.dataInput = xsDataInputMXWaitFileFirst self.waitFileFirst.executeSynchronous() if self.waitFileFirst.dataOutput.timedOut.value: strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % ( fWaitFileTimeout, firstFileName) #self.addWarningMessage(strWarningMessage) #self.WARNING(strWarningMessage) #self.sendMessageToMXCuBE(strWarningMessage, "error") edPluginControlReadImageHeader = self.loadPlugin( self.strEDPluginControlReadImageHeaderName) xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( XSDataString(firstFileName)) edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader edPluginControlReadImageHeader.executeSynchronous() subWedge = edPluginControlReadImageHeader.dataOutput.subWedge xsDataInputDozor = XSDataInputDozor() beam = subWedge.experimentalCondition.beam detector = subWedge.experimentalCondition.detector goniostat = subWedge.experimentalCondition.goniostat xsDataInputDozor.detectorType = detector.type xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value) xsDataInputDozor.spotSize = XSDataDouble(3.0) xsDataInputDozor.detectorDistance = XSDataDouble( detector.distance.value) xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value) orgx = detector.beamPositionY.value / detector.pixelSizeY.value orgy = detector.beamPositionX.value / detector.pixelSizeX.value xsDataInputDozor.orgx = XSDataDouble(orgx) xsDataInputDozor.orgy = XSDataDouble(orgy) # GB: the 50 might need tunig to CPU speed and number of. 2000 is a current limit of Dozor. self.maxChunkSize = 4000 #a min (150 * 2527 * 2463 /detector.numberPixelX.value / detector.numberPixelY.value, 2000) _beamstop = self.beamstop(detector) if _beamstop is not None: #self.WARNING("Setting beamstop shadow: %s"%_beamstop) xsDataInputDozor.ixMin = XSDataInteger(_beamstop['ix_min']) xsDataInputDozor.iyMin = XSDataInteger(_beamstop['iy_min']) xsDataInputDozor.ixMax = XSDataInteger(_beamstop['ix_max']) xsDataInputDozor.iyMax = XSDataInteger(_beamstop['iy_max']) if self.dataInput.pixelMin is not None: xsDataInputDozor.pixelMin = self.dataInput.pixelMin if self.dataInput.pixelMax is not None: xsDataInputDozor.pixelMax = self.dataInput.pixelMax _serial = 0 _startTime = time.time() chunk_list = self.schedule(goniostat.rotationAxisStart.value, goniostat.oscillationWidth.value) for chunk in chunk_list: if not self.poll_file( self.dataInput.template.value % (chunk['run_number'], chunk['first'] + chunk['number_of'] - 1), (beam.exposureTime.value + 0.003) * chunk['number_of'] + 30): self.sendMessageToMXCuBE("Timeout waiting for frame: %d" % (chunk['first'] + chunk['number_of']), level="error") return xsDataInputDozor.oscillationRange = XSDataDouble( chunk['rotation_range']) xsDataInputDozor.startingAngle = XSDataDouble( chunk['rotation_start']) xsDataInputDozor.firstImageNumber = XSDataInteger(chunk['first']) xsDataInputDozor.numberImages = XSDataInteger(chunk['number_of']) strFileName = self.dataInput.template.value % (chunk['run_number'], chunk['first']) strXDSTemplate = EDUtilsImage.getTemplate(strFileName, '?') xsDataInputDozor.nameTemplateImage = XSDataString( os.path.join(os.path.dirname(strFileName), strXDSTemplate)) edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor") edPluginDozor.dataInput = xsDataInputDozor edPluginDozor.executeSynchronous() xsDataChunkResultControlImageDozor = XSDataResultControlDozor() dozor_batch_list = [] #[[1,4,3,8,6], #[2, 5, 3, 3, 7],[1, 3, 12, 11, 4],[4, 2, 3, 5, 7]] #ozor_image_dict = {} diff_image_count = 0 xsDataControlImageDozor = XSDataControlImageDozor() for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor: xsDataControlImageDozor = XSDataControlImageDozor() xsDataControlImageDozor.number = xsDataResultDozor.number strFileName = self.dataInput.template.value % ( chunk['run_number'], xsDataControlImageDozor.number.value) xsDataControlImageDozor.image = XSDataFile( XSDataString(strFileName)) xsDataControlImageDozor.spots_num_of = xsDataResultDozor.spots_num_of xsDataControlImageDozor.spots_int_aver = xsDataResultDozor.spots_int_aver xsDataControlImageDozor.spots_resolution = xsDataResultDozor.spots_resolution xsDataControlImageDozor.powder_wilson_scale = xsDataResultDozor.powder_wilson_scale xsDataControlImageDozor.powder_wilson_bfactor = xsDataResultDozor.powder_wilson_bfactor xsDataControlImageDozor.powder_wilson_resolution = xsDataResultDozor.powder_wilson_resolution xsDataControlImageDozor.powder_wilson_correlation = xsDataResultDozor.powder_wilson_correlation xsDataControlImageDozor.powder_wilson_rfactor = xsDataResultDozor.powder_wilson_rfactor xsDataControlImageDozor.score = xsDataResultDozor.score xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor) dozor_batch_list.append( (xsDataControlImageDozor.number.getValue(), xsDataControlImageDozor.spots_num_of.getValue(), xsDataControlImageDozor.spots_int_aver.getValue(), xsDataControlImageDozor.spots_resolution.getValue(), xsDataControlImageDozor.score.getValue())) if xsDataControlImageDozor.spots_num_of.getValue() > 0: diff_image_count += 1 xsDataChunkResultControlImageDozor.addImageDozor( xsDataControlImageDozor) xsDataChunkResultControlImageDozor.exportToFile( "ResultControlDozor_Chunk_%06d.xml" % _serial) self.sendResultToMXCuBE(dozor_batch_list) _serial += 1 self.screen("Chunk %d/%d done in %.3f seconds" % \ (_serial, len(chunk_list), time.time()-_startTime)) self.sendMessageToMXCuBE("Chunk %d/%d done in %.2f sec., num diffr. frames: %d/%d" % \ (_serial, len(chunk_list), time.time() - _startTime, diff_image_count, len(dozor_batch_list) ) ) _startTime = time.time()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_3.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist" % dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist" % inp) time.sleep(1.0) shutil.copyfile(inp, dst) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], self.lstInput[1]]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.lstSummary.append("WARNING: " + self.strRadiationDamage) self.lstSummary.append("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) #outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin( self.__strControlledPluginAutoSub) base = "_".join( os.path.basename( self.__class__.lastSample.path.value).split("_") [:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) xsd = XSDataInputAutoSub( sampleCurve=self.__class__.lastSample, buffers=[ self.__class__.lastBuffer, self.dataInput.mergedCurve ], subtractedCurve=XSDataFile(XSDataString(sub))) self.__edPluginExecAutoSub.setDataInput(xsd) self.__edPluginExecAutoSub.connectSUCCESS( self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE( self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve self.__class__.lastSample = None else: self.__class__.lastSample = self.dataInput.mergedCurve
def getXSDataRaddoseInput(self, _xsDataBeam, _xsDataSample, _inumOperators): xsDataRaddoseInput = XSDataRaddoseInput() # Beam xsDataRaddoseInput.setBeamSize(_xsDataBeam.getSize()) xsDataRaddoseInput.setBeamFlux(_xsDataBeam.getFlux()) xsDataRaddoseInput.setBeamWavelength(_xsDataBeam.getWavelength()) xsDataRaddoseInput.setBeamExposureTime(XSDataTime(1.0)) xsDataRaddoseInput.setNumberOfImages(XSDataInteger(1)) xsDataRaddoseInput.setCrystalCell(_xsDataSample.getCrystal().getCell()) xsDataRaddoseInput.setCrystalSize(_xsDataSample.getSize()) xsDataComposition = _xsDataSample.getChemicalComposition() xsDataSolvent = xsDataComposition.getSolvent() if(xsDataSolvent is not None): xsDataRaddoseInput.setCrystalSATM(xsDataSolvent.getAtoms()) xsDataStructure = xsDataComposition.getStructure() if(xsDataStructure is not None): xsDataChains = xsDataStructure.getChain() totalNRESInStructure = 0 totalNDNAInStructure = 0 totalNRNAInStructure = 0 totalPATM = XSDataAtomicComposition() for chain in xsDataChains: # heavy atoms of each chain to be added in the PATM xsDataAtomicCompositionHeavyAtoms = chain.getHeavyAtoms() if (xsDataAtomicCompositionHeavyAtoms is not None): iterator = 1 while iterator <= chain.getNumberOfCopies().getValue(): totalPATM = self.mergeAtomicComposition(totalPATM, xsDataAtomicCompositionHeavyAtoms) iterator = iterator + 1 type = chain.getType().getValue() numberOfMonomers = chain.getNumberOfMonomers().getValue() * chain.getNumberOfCopies().getValue() if (type == "protein"): totalNRESInStructure = totalNRESInStructure + numberOfMonomers elif(type == "dna"): totalNDNAInStructure = totalNDNAInStructure + numberOfMonomers elif(type == "rna"): totalNRNAInStructure = totalNRNAInStructure + numberOfMonomers xsDataLigands = xsDataStructure.getLigand() for ligand in xsDataLigands: # Light atoms to be added to the NRES nres = ligand.getNumberOfLightAtoms().getValue() * ligand.getNumberOfCopies().getValue() / 7.85 totalNRESInStructure = totalNRESInStructure + nres # Heavy atoms to be added to the PATM if (ligand.getHeavyAtoms() is not None): iterator = 1 while iterator <= ligand.getNumberOfCopies().getValue(): totalPATM = self.mergeAtomicComposition(totalPATM, ligand.getHeavyAtoms()) iterator = iterator + 1 if(totalNRESInStructure != 0): xsDataRaddoseInput.setCrystalNRES(XSDataInteger(int(round(totalNRESInStructure)))) if(totalNDNAInStructure != 0): xsDataRaddoseInput.setCrystalNDNA(XSDataInteger(int(totalNDNAInStructure))) if(totalNRNAInStructure != 0): xsDataRaddoseInput.setCrystalNRNA(XSDataInteger(int(totalNRNAInStructure))) if(len(totalPATM.getAtom()) != 0): xsDataRaddoseInput.setCrystalPATM(totalPATM) xsDataNumberNumStructInAU = xsDataStructure.getNumberOfCopiesInAsymmetricUnit() xsDataNumberNumStructInUC = int(xsDataNumberNumStructInAU.getValue() * _inumOperators) xsDataRaddoseInput.setCrystalNMON(XSDataInteger(xsDataNumberNumStructInUC)) return xsDataRaddoseInput
def testIsSameExperimentalCondition(self): edPluginSubWedgeMergev1_1 = self.createPlugin() xsDataExperimentalConditionReference = self.getTestExperimentalCondition( ) xsDataExperimentalConditionSameAsReference = self.getTestExperimentalCondition( ) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionSameAsReference), True) xsDataExperimentalConditionDifferentExposureTime = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentExposureTime.getBeam( ).setExposureTime(XSDataTime(10.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentExposureTime), False) xsDataExperimentalConditionDifferentWavelength = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentWavelength.getBeam().setWavelength( XSDataWavelength(1.5)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentWavelength), False) xsDataExperimentalConditionDifferentBeamPositionX = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentBeamPositionX.getDetector( ).setBeamPositionX(XSDataLength(20.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentBeamPositionX), False) xsDataExperimentalConditionDifferentBeamPositionY = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentBeamPositionY.getDetector( ).setBeamPositionY(XSDataLength(20.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentBeamPositionY), False) xsDataExperimentalConditionDifferentDistance = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentDistance.getDetector().setDistance( XSDataLength(220.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentDistance), False) xsDataExperimentalConditionDifferentName = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentName.getDetector().setName( XSDataString(u"EDNA")) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentName), False) xsDataExperimentalConditionDifferentNumberPixelX = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentNumberPixelX.getDetector( ).setNumberPixelX(XSDataInteger(2)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentNumberPixelX), False) xsDataExperimentalConditionDifferentNumberPixelY = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentNumberPixelY.getDetector( ).setNumberPixelY(XSDataInteger(2)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentNumberPixelY), False) xsDataExperimentalConditionDifferentSerialNumber = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentSerialNumber.getDetector( ).setSerialNumber(XSDataString(u"EDNA")) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentSerialNumber), False) xsDataExperimentalConditionDifferentTwoTheta = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentTwoTheta.getDetector().setTwoTheta( XSDataAngle(90.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentTwoTheta), False) xsDataExperimentalConditionDifferentOscillationWidth = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentOscillationWidth.getGoniostat( ).setOscillationWidth(XSDataAngle(2.0)) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentOscillationWidth), False) xsDataExperimentalConditionDifferentRotationAxis = self.getTestExperimentalCondition( ) xsDataExperimentalConditionDifferentRotationAxis.getGoniostat( ).setRotationAxis(XSDataString(u"EDNA")) EDAssert.equal( edPluginSubWedgeMergev1_1.isSameExperimentalCondition( xsDataExperimentalConditionReference, xsDataExperimentalConditionDifferentRotationAxis), False)
def createInputCharacterisationFromSubWedges(self): EDVerbose.DEBUG( "EDPluginControlInterfacev1_2.createInputCharacterisationFromSubWedges" ) xsDataResultSubWedgeAssemble = self.edPluginControlSubWedgeAssemble.getDataOutput( ) self.xsDataInputCharacterisation = XSDataInputCharacterisation() xsDataCollection = XSDataCollection() # Default exposure time (for the moment, this value should be # possible to read from the command line) if self.xsDataDiffractionPlan is None: self.xsDataDiffractionPlan = XSDataDiffractionPlan() if (not xsDataResultSubWedgeAssemble is None): pyListSubWedge = xsDataResultSubWedgeAssemble.getSubWedge() xsDataCollection.setSubWedge(pyListSubWedge) for xsDataSubWedge in pyListSubWedge: if (self.strComplexity is not None): self.xsDataDiffractionPlan.setComplexity( XSDataString(self.strComplexity)) if (self.fFlux is not None): xsDataSubWedge.getExperimentalCondition().getBeam( ).setFlux(XSDataFlux(self.fFlux)) if (self.fBeamSizeX is not None) and (self.fBeamSizeY is not None): xsDataSize = XSDataSize() xsDataSize.setX(XSDataLength(self.fBeamSizeX)) xsDataSize.setY(XSDataLength(self.fBeamSizeY)) xsDataSubWedge.getExperimentalCondition().getBeam( ).setSize(xsDataSize) if (self.fBeamPosX is not None): xsDataSubWedge.getExperimentalCondition().getDetector( ).setBeamPositionX(XSDataLength(self.fBeamPosX)) if (self.fBeamPosY is not None): xsDataSubWedge.getExperimentalCondition().getDetector( ).setBeamPositionY(XSDataLength(self.fBeamPosY)) if (self.fMinExposureTimePerImage is not None): xsDataSubWedge.getExperimentalCondition().getBeam( ).setMinExposureTimePerImage( XSDataTime(self.fMinExposureTimePerImage)) if (self.fTransmission is not None): xsDataSubWedge.getExperimentalCondition().getBeam( ).setTransmission(XSDataDouble(self.fTransmission)) if (self.fWavelength is not None): xsDataSubWedge.getExperimentalCondition().getBeam( ).setWavelength(XSDataWavelength(self.fWavelength)) if self.fMinOscillationWidth != None: xsDataSubWedge.getExperimentalCondition().getGoniostat( ).setMinOscillationWidth( XSDataAngle(self.fMinOscillationWidth)) if self.fMaxOscillationSpeed != None: xsDataSubWedge.getExperimentalCondition().getGoniostat( ).setMaxOscillationSpeed( XSDataAngularSpeed(self.fMaxOscillationSpeed)) if (self.strForcedSpaceGroup is not None): self.xsDataDiffractionPlan.setForcedSpaceGroup( XSDataString(self.strForcedSpaceGroup)) self.xsDataDiffractionPlan.setAnomalousData( XSDataBoolean(self.bAnomalousData)) self.xsDataDiffractionPlan.setMaxExposureTimePerDataCollection( XSDataTime(self.fMaxExposureTimePerDataCollection)) if (self.strStrategyOption is not None): self.xsDataDiffractionPlan.setStrategyOption( XSDataString(self.strStrategyOption)) xsDataCollection.setDiffractionPlan(self.xsDataDiffractionPlan) if self.xsDataSample is not None: xsDataCollection.setSample( XSDataSampleCrystalMM.parseString(self.xsDataSample.marshal())) self.xsDataInputCharacterisation.setDataCollection(xsDataCollection)
def makeXML(self, filename): """Here we create the XML string to be passed to the EDNA plugin from the input filename This can / should be modified by the final user @param filename: full path of the input file @type filename: python string representing the path @rtype: XML string @return: python string """ self.header = None dirname, basename = os.path.split(filename) if not basename.startswith(self.prefix): return if self.normalizedSuffix and basename.endswith(self.normalizedSuffix): return if basename.startswith(self.flatPrefix): return if not basename.endswith(self.suffix): return if self.fScaleData: fScaleData = self.fScaleData else: fScaleData = self.getNbFrames(filename) if not fScaleData: fScaleData = 1.0 if self.fScaleDark: fScaleDark = self.fScaleDark else: fScaleDark = self.getNbFrames(self.darks[0]["path"]) if not fScaleDark: fScaleDark = 1.0 if isinstance(self.reference, int): reference = self.reference else: reference = -1 xsd = XSDataInputFullFieldXAS(HDF5File=XSDataFile(path=XSDataString(self.HDF5)), internalHDF5Path=XSDataString(self.internalHdf5), measureOffset=self.getXSDMeasureOffset(), dark=self.getXsdDark(), reference=XSDataInteger(reference), data=[XSDataImageExt(path=XSDataString(filename), exposureTime=XSDataTime(self.getExposureTime(filename)))], dataScaleFactor=XSDataDouble(fScaleData), darkScaleFactor=XSDataDouble(fScaleDark), ) if self.dontAlign: xsd.dontAlign = XSDataBoolean(self.dontAlign) extendedPrefix = "" number = "" started = False if self.lstSubscanSize: subScanDigit = [] for i in basename[len(self.prefix):]: if started and i == "_": if len(number) > 0: subScanDigit.append(number) number = "" continue if started and not i.isdigit(): if number: subScanDigit.append(number) number = "" break if not started: if i.isdigit(): started = True else: extendedPrefix += i if started: number += i if not subScanDigit: print("ERROR: no index guessed !!!") return "" elif len(subScanDigit) == 1: index = int(subScanDigit[0]) else:# len(subScanDigit) > 1: index = 0 for i in range(int(subScanDigit[0])): index += self.lstSubscanSize[i] index += int(subScanDigit[1]) extendedPrefix += "_".join(subScanDigit[:2]) else: for i in basename[len(self.prefix):]: extendedPrefix += i if started and not i.isdigit(): break if not started and i.isdigit(): started = True if started: number += i index = int(number) xsd.index = XSDataInteger(index) if self.normalizedSuffix: pr = os.path.splitext(os.path.abspath(filename))[0] xsd.saveNormalized = XSDataFile(path=XSDataString(pr + self.normalizedSuffix)) energy = self.getEnergy(filename) if energy is not None: xsd.energy = XSDataDouble(energy) flatprefix = self.flatPrefix + extendedPrefix listFlats = [] for oneFile in os.listdir(dirname): if oneFile.startswith(flatprefix) and oneFile.endswith(self.suffix): oneCompleteFile = os.path.abspath(os.path.join(dirname, oneFile)) xsdFileFlat1 = XSDataImageExt(path=XSDataString(oneCompleteFile), exposureTime=XSDataTime(self.getExposureTime(oneCompleteFile))) listFlats.append(xsdFileFlat1) xsd.flat = listFlats if len(listFlats) != 2: EDVerbose.WARNING("Not exactly 2 flats were found:" + " ".join([a.path.value for a in listFlats ])) EDVerbose.WARNING("Those are the flat prefix ans suffix: %s\t%s" % (flatprefix, self.suffix)) if self.fScaleFlat: fScaleFlat = self.fScaleFlat else: fScaleFlat = self.getNbFrames(oneCompleteFile) if not fScaleFlat: fScaleFlat = 1.0 xsd.flatScaleFactor = XSDataDouble(fScaleFlat) return xsd.marshal()
def testGetXSDataRaddoseInput(self): """ """ from XSDataCommon import XSDataLength from XSDataCommon import XSDataWavelength from XSDataCommon import XSDataFlux from XSDataCommon import XSDataSize from XSDataCommon import XSDataDouble from XSDataCommon import XSDataString from XSDataCommon import XSDataAngle from XSDataCommon import XSDataTime from XSDataCommon import XSDataInteger from XSDataMXv1 import XSDataBeam from XSDataMXv1 import XSDataStructure from XSDataMXv1 import XSDataChain from XSDataMXv1 import XSDataAtom from XSDataMXv1 import XSDataLigand from XSDataMXv1 import XSDataCrystal from XSDataMXv1 import XSDataSpaceGroup from XSDataMXv1 import XSDataSampleCrystalMM from XSDataMXv1 import XSDataChemicalCompositionMM from XSDataMXv1 import XSDataAtomicComposition from XSDataMXv1 import XSDataSolvent from XSDataMXv1 import XSDataCell from EDHandlerXSDataRaddosev10 import EDHandlerXSDataRaddosev10 EDFactoryPluginStatic.loadModule("XSDataRaddosev10") from XSDataRaddosev10 import XSDataRaddoseInput xsDataBeam = XSDataBeam() xsDataBeam.setSize(XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1))) xsDataBeam.setWavelength(XSDataWavelength(2.41)) xsDataBeam.setFlux(XSDataFlux(1e+12)) xsDataBeam.setExposureTime(XSDataTime(0.037)) xsDataSample = XSDataSampleCrystalMM() xsDataStructure = XSDataStructure() xsDataComposition = XSDataChemicalCompositionMM() xsDataChain = XSDataChain() xsDataChain.setType(XSDataString("protein")) xsDataChain.setNumberOfCopies(XSDataDouble(2)) xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtom1 = XSDataAtom() xsDataAtom1.setSymbol(XSDataString("Se")) xsDataAtom1.setNumberOf(XSDataDouble(4)) xsDataAtomicComposition.addAtom(xsDataAtom1) xsDataAtom2 = XSDataAtom() xsDataAtom2.setSymbol(XSDataString("S")) xsDataAtom2.setNumberOf(XSDataDouble(5)) xsDataAtomicComposition.addAtom(xsDataAtom2) xsDataChain.setHeavyAtoms(xsDataAtomicComposition) xsDataChain.setNumberOfMonomers(XSDataDouble(100)) xsDataStructure.addChain(xsDataChain) xsDataChain2 = XSDataChain() xsDataChain2.setType(XSDataString("rna")) xsDataChain2.setNumberOfCopies(XSDataDouble(1)) xsDataChain2.setNumberOfMonomers(XSDataDouble(60)) xsDataStructure.addChain(xsDataChain2) xsDataLigand = XSDataLigand() xsDataLigand.setNumberOfCopies(XSDataDouble(2)) xsDataLigand.setNumberOfLightAtoms(XSDataDouble(42)) xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtom3 = XSDataAtom() xsDataAtom3.setSymbol(XSDataString("Fe")) xsDataAtom3.setNumberOf(XSDataDouble(1)) xsDataAtomicComposition.addAtom(xsDataAtom3) xsDataLigand.setHeavyAtoms(xsDataAtomicComposition) xsDataStructure.addLigand(xsDataLigand) xsDataStructure.setNumberOfCopiesInAsymmetricUnit(XSDataDouble(0.25)) xsDataSolvent = XSDataSolvent() xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtomNa = XSDataAtom() xsDataAtomNa.setSymbol(XSDataString("Na")) xsDataAtomNa.setConcentration(XSDataDouble(1000)) xsDataAtomicComposition.addAtom(xsDataAtomNa) xsDataAtomCl = XSDataAtom() xsDataAtomCl.setSymbol(XSDataString("Cl")) xsDataAtomCl.setConcentration(XSDataDouble(1000)) xsDataAtomicComposition.addAtom(xsDataAtomCl) xsDataSolvent.setAtoms(xsDataAtomicComposition) xsDataComposition.setStructure(xsDataStructure) xsDataComposition.setSolvent(xsDataSolvent) xsDataSample.setChemicalComposition(xsDataComposition) xsDataSample.setSize( XSDataSize(XSDataLength(0.1), XSDataLength(0.1), XSDataLength(0.1))) xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0), angle_beta=XSDataAngle(90.0), angle_gamma=XSDataAngle(90.0), length_a=XSDataLength(78.9), length_b=XSDataLength(95.162), length_c=XSDataLength(104.087)) xsDataCrystal = XSDataCrystal() xsDataSpaceGroup = XSDataSpaceGroup() xsDataCrystal.setCell(xsDataCell) xsDataSpaceGroup.setITNumber(XSDataInteger(16)) xsDataCrystal.setSpaceGroup(xsDataSpaceGroup) xsDataSample.setCrystal(xsDataCrystal) iNumSymOperators = 4 iNumberOfImages = 2 xsDataRaddosev01Input = EDHandlerXSDataRaddosev10( ).getXSDataRaddoseInput(xsDataBeam, xsDataSample, iNumSymOperators, iNumberOfImages) xsDataRaddosev01Input.exportToFile(self.strObtainedInputFile2) strExpectedInput = EDUtilsTest.readAndParseFile( self.strReferenceInputFile2) strObtainedInput = EDUtilsTest.readAndParseFile( self.strObtainedInputFile2) xsDataInputExpected = XSDataRaddoseInput.parseString(strExpectedInput) xsDataInputObtained = XSDataRaddoseInput.parseString(strObtainedInput) EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal())
def testSetDataModelInput(self): edPluginBest = self.createPlugin() edConfigurationGood01 = EDConfiguration( os.path.join(self.strDataPath, "XSConfiguration.xml")) dictItemGood01 = edConfigurationGood01.get(self.getPluginName()) edPluginBest.setConfig(dictItemGood01, _bLocal=True) edPluginBest.setScriptExecutable("cat") edPluginBest.configure() from XSDataBestv1_3 import XSDataInputBest xsDataInputBest = XSDataInputBest() from XSDataCommon import XSDataAbsorbedDoseRate from XSDataCommon import XSDataDouble from XSDataCommon import XSDataString from XSDataCommon import XSDataTime from XSDataCommon import XSDataFile from XSDataCommon import XSDataAngularSpeed from XSDataCommon import XSDataString from XSDataCommon import XSDataAngle from XSDataCommon import XSDataBoolean xsDataInputBest.setCrystalAbsorbedDoseRate( XSDataAbsorbedDoseRate(0.22E+06)) xsDataInputBest.setCrystalShape(XSDataDouble(1)) xsDataInputBest.setCrystalSusceptibility(XSDataDouble(1.5)) xsDataInputBest.setDetectorType(XSDataString("q210-2x")) xsDataInputBest.setBeamExposureTime(XSDataTime(1)) xsDataInputBest.setBeamMaxExposureTime(XSDataTime(10000)) xsDataInputBest.setBeamMinExposureTime(XSDataTime(0.1)) xsDataInputBest.setGoniostatMinRotationWidth(XSDataAngle(0.1)) xsDataInputBest.setGoniostatMaxRotationSpeed(XSDataAngularSpeed(10)) xsDataInputBest.setAimedResolution(XSDataDouble(2)) xsDataInputBest.setAimedRedundancy(XSDataDouble(6.5)) xsDataInputBest.setAimedCompleteness(XSDataDouble(0.9)) xsDataInputBest.setAimedIOverSigma(XSDataDouble(3)) xsDataInputBest.setComplexity(XSDataString("min")) xsDataInputBest.setAnomalousData(XSDataBoolean(False)) fileDirectory = edPluginBest.getWorkingDirectory() bestFileContentDat = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile.dat")) xsDataInputBest.setBestFileContentDat(XSDataString(bestFileContentDat)) bestFileContentPar = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile.par")) xsDataInputBest.setBestFileContentPar(XSDataString(bestFileContentPar)) bestFileContentHKL = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile1.hkl")) xsDataInputBest.addBestFileContentHKL(XSDataString(bestFileContentHKL)) xsDataInputBest.exportToFile(self.obtainedInputFile) pyStrExpectedInput = self.readAndParseFile(self.referenceInputFile) pyStrObtainedInput = self.readAndParseFile(self.obtainedInputFile) xsDataInputExpected = XSDataInputBest.parseString(pyStrExpectedInput) xsDataInputObtained = XSDataInputBest.parseString(pyStrObtainedInput) EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal()) EDUtilsFile.deleteFile(self.obtainedInputFile) self.cleanUp(edPluginBest)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_2.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: shutil.copyfile(self.lstInput[0].path.value, self.dataInput.mergedCurve.path.value) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], self.lstInput[1]]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.lstSummary.append("WARNING: " + self.strRadiationDamage) self.lstSummary.append("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) #outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() self.dictAve[self.tKey] = self.dataInput.mergedCurve if len(self.tKey) == 3 and (self.tKey[0], self.tKey[1], 0.0) in self.dictAve: #try to subtract buffer automatically bufferKey = (self.tKey[0], self.tKey[1], 0.0) bufferPath = self.dictAve[bufferKey] if self.strSubFile is None: self.strSubFile = self.strMergedFile.replace( "_ave.dat", "_sub.dat") if self.strMergedFile == self.strSubFile: self.strSubFile = None return xsdSubFile = XSDataFile(XSDataString(self.strSubFile)) xsdSub = XSDataInputDatop( inputCurve=[self.dataInput.mergedCurve, bufferPath], outputCurve=xsdSubFile, operation=XSDataString("SUB")) self.lstSub.append("Merged data from : %s" % self.dataInput.mergedCurve.path.value) self.lstSub.append("Subtracted buffer: %s" % bufferPath.path.value) self.__edPluginExecDataop = self.loadPlugin( self.__strControlledPluginDatop) self.__edPluginExecDataop.setDataInput(xsdSub) self.__edPluginExecDataop.connectSUCCESS( self.doSuccessExecDatop) self.__edPluginExecDataop.connectFAILURE( self.doFailureExecDatop) self.__edPluginExecDataop.executeSynchronous() if self.isFailure(): return elif self.strSubFile is not None: self.__edPluginExecAutoRg = self.loadPlugin( self.__strControlledPluginAutoRG) xsd = XSDataInputAutoRg(inputCurve=[xsdSubFile]) self.__edPluginExecAutoRg.setDataInput(xsd) self.__edPluginExecAutoRg.connectSUCCESS( self.doSuccessExecAutoRg) self.__edPluginExecAutoRg.connectFAILURE( self.doFailureExecAutoRg) self.__edPluginExecAutoRg.executeSynchronous() self.rewriteHeader(self.strSubFile, self.strSubFile) else: self.__edPluginExecAutoRg = self.loadPlugin( self.__strControlledPluginAutoRG) xsd = XSDataInputAutoRg( inputCurve=[self.dataInput.mergedCurve]) self.__edPluginExecAutoRg.setDataInput(xsd) self.__edPluginExecAutoRg.connectSUCCESS( self.doSuccessExecAutoRg) self.__edPluginExecAutoRg.connectFAILURE( self.doFailureExecAutoRg) self.__edPluginExecAutoRg.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_0.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() # if self.isFailure(): # return if len(self.lstInput) == 1: shutil.copyfile(self.lstInput[0].path.value, self.dataInput.mergedCurve.path.value) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for oneFile in self.lstInput[1:]: edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None) and (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstSummary.append("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous()
def setPluginInput(self, _edPlugin): xsDataDiffractionPlan = XSDataDiffractionPlan() if (not self.__fMaxExposureTimePerDataCollection is None): xsDataDiffractionPlan.setMaxExposureTimePerDataCollection( XSDataTime(self.__fMaxExposureTimePerDataCollection)) if (not self.__strForcedSpaceGroup is None): xsDataDiffractionPlan.setForcedSpaceGroup( XSDataString(self.__strForcedSpaceGroup)) if (not self.__bAnomalousData is None): xsDataDiffractionPlan.setAnomalousData( XSDataBoolean(self.__bAnomalousData)) if (not self.__strStrategyOption is None): xsDataDiffractionPlan.setStrategyOption( XSDataString(self.__strStrategyOption)) if (not self.__fDoseLimit is None): xsDataDiffractionPlan.setDoseLimit(XSDataDouble(self.__fDoseLimit)) if (not self.__fAimedCompleteness is None): xsDataDiffractionPlan.setAimedCompleteness( XSDataDouble(self.__fAimedCompleteness)) if (not self.__fAimedMultiplicity is None): xsDataDiffractionPlan.setAimedMultiplicity( XSDataDouble(self.__fAimedMultiplicity)) if (not self.__fAimedResolution is None): xsDataDiffractionPlan.setAimedResolution( XSDataDouble(self.__fAimedResolution)) if (not self.__strComplexity is None): xsDataDiffractionPlan.setComplexity( XSDataString(self.__strComplexity)) _edPlugin.setDataInput(xsDataDiffractionPlan, "diffractionPlan") if (not self.__listImagePaths is None): for strImagePath in self.__listImagePaths: _edPlugin.setDataInput(XSDataString(strImagePath), "imagePaths") if (not self.__xsDataInputCharacterisation is None): _edPlugin.setDataInput(self.__xsDataInputCharacterisation, "inputCharacterisation") if (not self.__fFlux is None): _edPlugin.setDataInput(XSDataFloat(self.__fFlux), "flux") if (not self.__fMinExposureTimePerImage is None): _edPlugin.setDataInput( XSDataFloat(self.__fMinExposureTimePerImage), "minExposureTimePerImage") if (not self.__fBeamSize is None): _edPlugin.setDataInput(XSDataFloat(self.__fBeamSize), "beamSize") if (not self.__bTemplateMode is None): _edPlugin.setDataInput(XSDataBoolean(self.__bTemplateMode), "templateMode") if (not self.__strGeneratedTemplateFile is None): _edPlugin.setDataInput( XSDataString(self.__strGeneratedTemplateFile), "generatedTemplateFile") if (not self.__strResultsFilePath is None): _edPlugin.setDataInput(XSDataString(self.__strResultsFilePath), "resultsFilePath") if (not self.__fBeamPosX is None): _edPlugin.setDataInput(XSDataFloat(self.__fBeamPosX), "beamPosX") if (not self.__fBeamPosY is None): _edPlugin.setDataInput(XSDataFloat(self.__fBeamPosY), "beamPosY") if (not self.__iDataCollectionId is None): _edPlugin.setDataInput(XSDataInteger(self.__iDataCollectionId), "dataCollectionId") if (not self.__strShortComments is None): _edPlugin.setDataInput(XSDataString(self.__strShortComments), "shortComments") if (not self.__strComments is None): _edPlugin.setDataInput(XSDataString(self.__strComments), "comments") if (not self.__fTransmission is None): _edPlugin.setDataInput(XSDataDouble(self.__fTransmission), "transmission")
def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("EDPluginExecReadImageHeaderPilatus6Mv10.process") xsDataInputReadImageHeader = self.getDataInput() xsDataFile = xsDataInputReadImageHeader.getImage() strPath = xsDataFile.getPath().getValue() dictPilatus6MHeader = self.readHeaderPilatus6M(strPath) if (dictPilatus6MHeader is None): strErrorMessage = "EDPluginExecReadImageHeaderPilatus6Mv10.process : Cannot read header : %s" % strPath self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() else: xsDataExperimentalCondition = XSDataExperimentalCondition() xsDataDetector = XSDataDetector() iNoPixelsX = 2463 iNoPixelsY = 2527 xsDataDetector.setNumberPixelX(XSDataInteger(iNoPixelsX)) xsDataDetector.setNumberPixelY(XSDataInteger(iNoPixelsY)) # Pixel size listPixelSizeXY = dictPilatus6MHeader["Pixel_size"].split(" ") fPixelSizeX = float(listPixelSizeXY[0]) * 1000 xsDataDetector.setPixelSizeX(XSDataLength(fPixelSizeX)) fPixelSizeY = float(listPixelSizeXY[3]) * 1000 xsDataDetector.setPixelSizeY(XSDataLength(fPixelSizeY)) # Beam position listBeamPosition = dictPilatus6MHeader["Beam_xy"].replace( "(", " ").replace(")", " ").replace(",", " ").split() fBeamPositionX = float(listBeamPosition[1]) * fPixelSizeX fBeamPositionY = float(listBeamPosition[0]) * fPixelSizeY xsDataDetector.setBeamPositionX(XSDataLength(fBeamPositionX)) xsDataDetector.setBeamPositionY(XSDataLength(fBeamPositionY)) fDistance = float( dictPilatus6MHeader["Detector_distance"].split(" ")[0]) * 1000 xsDataDetector.setDistance(XSDataLength(fDistance)) # xsDataDetector.setNumberBytesInHeader(XSDataInteger(float(dictPilatus6MHeader[ "header_size" ]))) xsDataDetector.setSerialNumber( XSDataString(dictPilatus6MHeader["Detector:"])) # #xsDataDetector.setBin( XSDataString( dictPilatus6MHeader[ "BIN" ] ) ) ) # #xsDataDetector.setDataType( XSDataString( dictPilatus6MHeader[ "TYPE" ] ) ) ) # #xsDataDetector.setByteOrder( XSDataString( dictPilatus6MHeader[ "BYTE_ORDER" ] ) ) ) # xsDataDetector.setImageSaturation(XSDataInteger(int(dictPilatus6MHeader[ "saturation_level" ]))) xsDataDetector.setName(XSDataString("PILATUS 6M")) xsDataDetector.setType(XSDataString("pilatus6m")) xsDataExperimentalCondition.setDetector(xsDataDetector) # Beam object xsDataBeam = XSDataBeam() xsDataBeam.setWavelength( XSDataWavelength( float(dictPilatus6MHeader["Wavelength"].split(" ")[0]))) xsDataBeam.setExposureTime( XSDataTime( float(dictPilatus6MHeader["Exposure_time"].split(" ")[0]))) xsDataExperimentalCondition.setBeam(xsDataBeam) # Goniostat object xsDataGoniostat = XSDataGoniostat() fRotationAxisStart = float( dictPilatus6MHeader["Start_angle"].split(" ")[0]) fOscillationWidth = float( dictPilatus6MHeader["Angle_increment"].split(" ")[0]) if "Kappa" in dictPilatus6MHeader.keys(): fKappa = float(dictPilatus6MHeader["Kappa"].split(" ")[0]) xsDataGoniostat.setKappa(XSDataAngle(fKappa)) if "Phi" in dictPilatus6MHeader.keys(): fPhi = float(dictPilatus6MHeader["Phi"].split(" ")[0]) xsDataGoniostat.setPhi(XSDataAngle(fPhi)) xsDataGoniostat.setRotationAxisStart( XSDataAngle(fRotationAxisStart)) xsDataGoniostat.setRotationAxisEnd( XSDataAngle(fRotationAxisStart + fOscillationWidth)) xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth)) xsDataExperimentalCondition.setGoniostat(xsDataGoniostat) # # Create the image object xsDataImage = XSDataImage() xsDataImage.setPath(XSDataString(strPath)) if "DateTime" in dictPilatus6MHeader: strTimeStamp = dictPilatus6MHeader["DateTime"] xsDataImage.setDate(XSDataString(strTimeStamp)) iImageNumber = EDUtilsImage.getImageNumber(strPath) xsDataImage.setNumber(XSDataInteger(iImageNumber)) xsDataSubWedge = XSDataSubWedge() xsDataSubWedge.setExperimentalCondition( xsDataExperimentalCondition) xsDataSubWedge.addImage(xsDataImage) self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader() self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def testSetDataModelInput(self): """ """ edPluginBest = self.createPlugin() xsPluginItemGood01 = self.getPluginConfiguration( os.path.join(self.strDataPath, "XSConfiguration.xml")) edPluginBest.setConfiguration(xsPluginItemGood01) edPluginBest.setScriptExecutable("cat") edPluginBest.configure() from XSDataBestv1_1 import XSDataInputBest xsDataInputBest = XSDataInputBest() from XSDataCommon import XSDataAbsorbedDoseRate from XSDataCommon import XSDataDouble from XSDataCommon import XSDataTime from XSDataCommon import XSDataSpeed from XSDataCommon import XSDataString from XSDataCommon import XSDataAngle from XSDataCommon import XSDataBoolean xsDataInputBest.setCrystalAbsorbedDoseRate( XSDataAbsorbedDoseRate(0.22E+06)) xsDataInputBest.setCrystalShape(XSDataDouble(1)) xsDataInputBest.setCrystalSusceptibility(XSDataDouble(1.5)) xsDataInputBest.setDetectorType(XSDataString("q210-2x")) xsDataInputBest.setBeamExposureTime(XSDataTime(1)) xsDataInputBest.setBeamMaxExposureTime(XSDataTime(10000)) xsDataInputBest.setBeamMinExposureTime(XSDataTime(0.1)) xsDataInputBest.setGoniostatMinRotationWidth(XSDataAngle(0.1)) xsDataInputBest.setGoniostatMaxRotationSpeed(XSDataSpeed(10)) xsDataInputBest.setAimedResolution(XSDataDouble(2)) xsDataInputBest.setAimedRedundancy(XSDataDouble(6.5)) xsDataInputBest.setAimedCompleteness(XSDataDouble(0.9)) xsDataInputBest.setAimedIOverSigma(XSDataDouble(3)) xsDataInputBest.setComplexity(XSDataString("min")) xsDataInputBest.setAnomalousData(XSDataBoolean(True)) bestFileContentDat = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile.dat")) xsDataInputBest.setBestFileContentDat(XSDataString(bestFileContentDat)) bestFileContentPar = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile.par")) xsDataInputBest.setBestFileContentPar(XSDataString(bestFileContentPar)) bestFileContentHKL = EDUtilsFile.readFile( os.path.join(self.strDataPath, "bestfile1.hkl")) listBestFileContentHKL = [] listBestFileContentHKL.append(XSDataString(bestFileContentHKL)) xsDataInputBest.setBestFileContentHKL(listBestFileContentHKL) xsDataInputBest.outputFile(self.strObtainedInputFile) strExpectedInput = self.readAndParseFile(self.strReferenceInputFile) strObtainedInput = self.readAndParseFile(self.strObtainedInputFile) xsDataInputExpected = XSDataInputBest.parseString(strExpectedInput) xsDataInputObtained = XSDataInputBest.parseString(strObtainedInput) EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal()) os.remove(self.strObtainedInputFile) self.cleanUp(edPluginBest)
def testSetDataModelInput(self): """ """ edPluginStrategy = self.createPlugin() xsPluginItemGood01 = self.getPluginConfiguration(os.path.join(self.strDataPath, "XSConfiguration_ESRF.xml")) edPluginStrategy.setConfiguration(xsPluginItemGood01) edPluginStrategy.configure() from XSDataMXv1 import XSDataStrategyInput xSDataStrategy = XSDataStrategyInput() # Beam from XSDataCommon import XSDataFlux from XSDataCommon import XSDataWavelength from XSDataCommon import XSDataSize from XSDataCommon import XSDataLength from XSDataCommon import XSDataTime from XSDataMXv1 import XSDataBeam from XSDataMXv1 import XSDataExperimentalCondition xsExperimentalCondition = XSDataExperimentalCondition() xsBeam = XSDataBeam() xsBeam.setFlux(XSDataFlux(1e+12)) xsBeam.setWavelength(XSDataWavelength(2.41)) xsBeam.setSize(XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1))) xsBeam.setExposureTime(XSDataTime(1)) xsExperimentalCondition.setBeam(xsBeam) # Detector and Exposure Time from XSDataMXv1 import XSDataDetector from XSDataCommon import XSDataString from XSDataMXv1 import XSDataGoniostat xsDataDetector = XSDataDetector() xsDataDetector.setType(XSDataString("q210-2x")) xsExperimentalCondition.setDetector(xsDataDetector) xsDataGoniostat = XSDataGoniostat() xsDataGoniostat.setRotationAxis(XSDataString("phi")) xsExperimentalCondition.setGoniostat(xsDataGoniostat) xSDataStrategy.setExperimentalCondition(xsExperimentalCondition) # Best Files bestFileContentDat = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.dat")) xSDataStrategy.setBestFileContentDat(XSDataString(bestFileContentDat)) bestFileContentPar = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.par")) xSDataStrategy.setBestFileContentPar(XSDataString(bestFileContentPar)) bestFileContentHKL = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile1.hkl")) listBestFileContentHKL = [] listBestFileContentHKL.append(XSDataString(bestFileContentHKL)) xSDataStrategy.setBestFileContentHKL(listBestFileContentHKL) # Crystal from XSDataCommon import XSDataFloat from XSDataCommon import XSDataAngle from XSDataCommon import XSDataInteger from XSDataMXv1 import XSDataCrystal from XSDataMXv1 import XSDataStructure from XSDataMXv1 import XSDataChain from XSDataMXv1 import XSDataAtom from XSDataMXv1 import XSDataLigand from XSDataMXv1 import XSDataSampleCrystalMM from XSDataMXv1 import XSDataChemicalCompositionMM from XSDataMXv1 import XSDataAtomicComposition from XSDataMXv1 import XSDataSolvent from XSDataMXv1 import XSDataCell from XSDataMXv1 import XSDataSpaceGroup xsDataSampleCrystalMM = XSDataSampleCrystalMM() xsDataStructure = XSDataStructure() xsDataComposition = XSDataChemicalCompositionMM() xsDataChain = XSDataChain() xsDataChain.setType(XSDataString("protein")) xsDataChain.setNumberOfCopies(XSDataFloat(2)) xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtom1 = XSDataAtom() xsDataAtom1.setSymbol(XSDataString("Se")) xsDataAtom1.setNumberOf(XSDataFloat(4)) xsDataAtomicComposition.addAtom(xsDataAtom1) xsDataChain.setHeavyAtoms(xsDataAtomicComposition) xsDataChain.setNumberOfMonomers(XSDataFloat(100)) xsDataStructure.addChain(xsDataChain) xsDataChain2 = XSDataChain() xsDataChain2.setType(XSDataString("rna")) xsDataChain2.setNumberOfCopies(XSDataFloat(1)) xsDataChain2.setNumberOfMonomers(XSDataFloat(60)) xsDataStructure.addChain(xsDataChain2) xsDataLigand = XSDataLigand() xsDataLigand.setNumberOfCopies(XSDataFloat(2)) xsDataLigand.setNumberOfLightAtoms(XSDataFloat(42)) xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtom2 = XSDataAtom() xsDataAtom2.setSymbol(XSDataString("Fe")) xsDataAtom2.setNumberOf(XSDataFloat(1)) xsDataAtomicComposition.addAtom(xsDataAtom2) xsDataLigand.setHeavyAtoms(xsDataAtomicComposition) xsDataStructure.addLigand(xsDataLigand) xsDataStructure.setNumberOfCopiesInAsymmetricUnit(XSDataFloat(0.25)) xsDataSolvent = XSDataSolvent() xsDataAtomicComposition = XSDataAtomicComposition() xsDataAtom3 = XSDataAtom() xsDataAtom3.setSymbol(XSDataString("Na")) xsDataAtom3.setConcentration(XSDataFloat(1000)) xsDataAtom4 = XSDataAtom() xsDataAtom4.setSymbol(XSDataString("Cl")) xsDataAtom4.setConcentration(XSDataFloat(1000)) xsDataAtomicComposition.addAtom(xsDataAtom3) xsDataAtomicComposition.addAtom(xsDataAtom4) xsDataSolvent.setAtoms(xsDataAtomicComposition) xsDataComposition.setStructure(xsDataStructure) xsDataComposition.setSolvent(xsDataSolvent) xsDataSampleCrystalMM.setChemicalComposition(xsDataComposition) xsDataSampleCrystalMM.setSize(XSDataSize(XSDataLength(0.1), XSDataLength(0.1), XSDataLength(0.1))) xsDataCrystal = XSDataCrystal() xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0), angle_beta=XSDataAngle(90.0), angle_gamma=XSDataAngle(90.0), length_a=XSDataLength(78.9), length_b=XSDataLength(95.162), length_c=XSDataLength(104.087)) xsDataCrystal.setCell(xsDataCell) xsDataSpaceGroup = XSDataSpaceGroup() xsDataSpaceGroup.setITNumber(XSDataInteger(16)) xsDataCrystal.setSpaceGroup(xsDataSpaceGroup) xsDataSampleCrystalMM.setSusceptibility(XSDataFloat(1.5)) xSDataStrategy.setCrystalRefined(xsDataCrystal) xSDataStrategy.setSample(xsDataSampleCrystalMM) xSDataStrategy.outputFile(self.strObtainedInputFile) strExpectedInput = self.readAndParseFile (self.strReferenceInputFile) strObtainedInput = self.readAndParseFile (self.strObtainedInputFile) xsDataInputExpected = XSDataStrategyInput.parseString(strExpectedInput) xsDataInputObtained = XSDataStrategyInput.parseString(strObtainedInput) EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal())
def getDataCollectionOutputDataFromLog(self, _strBestLog): xsDataResultBest = XSDataResultBest() # loop through the lines of the log file isScanningWedge = False indexLine = 0 listLog = _strBestLog.split("\n") iCollectionPlanNumber = 1 while indexLine < len(listLog): if "Plan of data collection for radiation damage characterisation" in listLog[ indexLine]: xsDataBestCollectionPlan = XSDataBestCollectionPlan() xsDataBestStrategySummary = XSDataBestStrategySummary() indexLine += 2 regEx = re.compile( """ Resolution limit =\s*(\d+\.\d+) Angstrom\s*Distance =\s*(\d+\.\d+)mm""" ) matchObj = regEx.search(listLog[indexLine]) if matchObj == None: raise BaseException("Cannot parse best log file!") resultMatch = matchObj.groups() resultion = float(resultMatch[0]) distance = float(resultMatch[1]) xsDataBestStrategySummary.resolution = XSDataDouble(resultion) xsDataBestStrategySummary.distance = XSDataLength(distance) xsDataBestCollectionPlan.strategySummary = xsDataBestStrategySummary # Burn strategy indexLine += 7 collectionRunNumber = 1 while not listLog[indexLine].startswith( "-------------------------------"): listLine = listLog[indexLine].split() if listLine[0].startswith( "exposure") or listLine[0].startswith("burn"): xsDataBestCollectionRun = XSDataBestCollectionRun() xsDataBestCollectionRun.collectionRunNumber = XSDataInteger( collectionRunNumber) xsDataBestCollectionRun.action = XSDataString( listLine[0]) xsDataBestCollectionRun.phiStart = XSDataAngle( listLine[1]) xsDataBestCollectionRun.phiWidth = XSDataAngle( listLine[2]) xsDataBestCollectionRun.exposureTime = XSDataTime( listLine[3]) xsDataBestCollectionRun.numberOfImages = XSDataInteger( listLine[4].split("|")[0]) xsDataBestCollectionRun.transmission = XSDataDouble( listLine[5]) xsDataBestCollectionPlan.addCollectionRun( xsDataBestCollectionRun) collectionRunNumber += 1 indexLine += 1 xsDataBestCollectionPlan.collectionPlanNumber = XSDataInteger( iCollectionPlanNumber) xsDataResultBest.addCollectionPlan(xsDataBestCollectionPlan) if "Main Wedge" in listLog[indexLine] or "Low resolution Wedge" in listLog[indexLine] \ or "Strategy for SAD data collection" in listLog[indexLine]: isScanningWedge = True xsDataBestCollectionPlan = XSDataBestCollectionPlan() xsDataBestStrategySummary = XSDataBestStrategySummary() if "Main Wedge" in listLog[indexLine]: indexLine += 2 # ResolutionReasoning xsDataBestStrategySummary.resolutionReasoning = XSDataString( listLog[indexLine].strip()) elif "Strategy for SAD data collection" in listLog[indexLine]: indexLine += 2 xsDataBestStrategySummary.resolutionReasoning = XSDataString( listLog[indexLine].strip()) else: xsDataBestStrategySummary.resolutionReasoning = XSDataString( "Low-resolution pass, no overloads and full completeness" ) # Resolution, transmission, distance indexLine += 1 while not "Resolution limit" in listLog[indexLine]: indexLine += 1 regEx = re.compile( """ Resolution limit =\s*(\d+\.\d+) Angstrom Transmission =\s*(\d+\.\d+)% Distance =\s*(\d+\.\d+)mm""" ) matchObj = regEx.search(listLog[indexLine]) if matchObj == None: raise BaseException("Cannot parse best log file!") resultMatch = matchObj.groups() resultion = float(resultMatch[0]) transmission = float(resultMatch[1]) distance = float(resultMatch[2]) xsDataBestStrategySummary.resolution = XSDataDouble(resultion) xsDataBestStrategySummary.transmission = XSDataDouble( transmission) xsDataBestStrategySummary.distance = XSDataLength(distance) # Resolution, transmission, distance indexLine += 8 while not listLog[indexLine].startswith( "-------------------------------"): listLine = listLog[indexLine].split() xsDataBestCollectionRun = XSDataBestCollectionRun() xsDataBestCollectionRun.collectionRunNumber = XSDataInteger( listLine[0]) xsDataBestCollectionRun.phiStart = XSDataAngle(listLine[1]) xsDataBestCollectionRun.phiWidth = XSDataAngle(listLine[2]) xsDataBestCollectionRun.exposureTime = XSDataTime( listLine[3]) xsDataBestCollectionRun.numberOfImages = XSDataInteger( listLine[4].split("|")[0]) xsDataBestCollectionRun.overlaps = XSDataString( listLine[5]) xsDataBestCollectionPlan.addCollectionRun( xsDataBestCollectionRun) indexLine += 1 # xsDataBestStrategySummary.completeness = XSDataDouble( listLine[11]) # while not "Redundancy" in listLog[indexLine]: indexLine += 1 xsDataBestStrategySummary.redundancy = XSDataDouble( listLog[indexLine].split()[-1]) # while not "I/Sigma (outer shell)" in listLog[indexLine]: indexLine += 1 xsDataBestStrategySummary.iSigma = XSDataDouble( listLog[indexLine].split()[6].replace(")", "")) # while not "Total Exposure time" in listLog[indexLine]: indexLine += 1 xsDataBestStrategySummary.totalExposureTime = XSDataTime( listLog[indexLine].split()[4]) # while not "Total Data Collection time" in listLog[indexLine]: indexLine += 1 xsDataBestStrategySummary.totalDataCollectionTime = XSDataTime( listLog[indexLine].split()[5]) # while not "Wedge Data Collection Statistics according to the Strategy" in listLog[ indexLine]: indexLine += 1 (xsDataBestStatisticalPrediction, indexLine) = self.getXSDataBestStatisticalPrediction( listLog, indexLine) xsDataBestCollectionPlan.strategySummary = xsDataBestStrategySummary xsDataBestCollectionPlan.statisticalPrediction = xsDataBestStatisticalPrediction xsDataBestCollectionPlan.collectionPlanNumber = XSDataInteger( iCollectionPlanNumber) iCollectionPlanNumber += 1 xsDataResultBest.addCollectionPlan(xsDataBestCollectionPlan) if "Additional information" in listLog[indexLine]: # while not "Relative scale" in listLog[indexLine]: indexLine += 1 scale = float(listLog[indexLine].split()[-1]) # while not "Overall B-factor" in listLog[indexLine]: indexLine += 1 # bFactor = float(listLog[indexLine].split()[-2]) while not "Estimated limit of resolution" in listLog[indexLine]: indexLine += 1 rankingResolution = float(listLog[indexLine].split()[5]) for collectionPlan in xsDataResultBest.collectionPlan: xsDataCrystalScale = XSDataCrystalScale() xsDataCrystalScale.scale = XSDataDouble(scale) xsDataCrystalScale.bFactor = XSDataDouble(bFactor) collectionPlan.crystalScale = xsDataCrystalScale collectionPlan.strategySummary.rankingResolution = XSDataDouble( rankingResolution) indexLine += 1 return xsDataResultBest