def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputWaitFile = XSDataInputWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: if not os.path.exists(xsDataImage.path.value): self.screen("Waiting for image %s" % xsDataImage.path.value) self.edPluginWaitFile = self.loadPlugin( self.strPluginWaitFileName) xsDataInputWaitFile.expectedFile = XSDataFile(xsDataImage.path) xsDataInputWaitFile.setExpectedSize(XSDataInteger(100000)) xsDataInputWaitFile.setTimeOut( XSDataTime(self.fWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fWaitFileTimeOut) self.edPluginWaitFile.setDataInput(xsDataInputWaitFile) self.edPluginWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) listPlugin.append(edPluginPluginExecImageQualityIndicator) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for edPluginPluginExecImageQualityIndicator in listPlugin: edPluginPluginExecImageQualityIndicator.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def preProcess(self, _edObject = None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict(creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format(self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath(os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath(os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputWaitFile = XSDataInputWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPlugin = [] for xsDataImage in listXSDataImage: if not os.path.exists(xsDataImage.path.value): self.screen("Waiting for image %s" % xsDataImage.path.value) self.edPluginWaitFile = self.loadPlugin(self.strPluginWaitFileName) xsDataInputWaitFile.expectedFile = XSDataFile(xsDataImage.path) xsDataInputWaitFile.setExpectedSize(XSDataInteger(100000)) xsDataInputWaitFile.setTimeOut(XSDataTime(self.fWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fWaitFileTimeOut) self.edPluginWaitFile.setDataInput(xsDataInputWaitFile) self.edPluginWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() edPluginControlBackground3D = self.loadPlugin(self.strPluginNameControlBackground3D) listPlugin.append([edPluginPluginExecImageQualityIndicator, edPluginControlBackground3D]) xsDataInputControlBackground3D = XSDataInputControlBackground3D() xsDataInputControlBackground3D.addImage(XSDataFile(xsDataImage.path)) edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D edPluginControlBackground3D.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for pluginPair in listPlugin: edPluginPluginExecImageQualityIndicator = pluginPair[0] edPluginControlBackground3D = pluginPair[1] edPluginPluginExecImageQualityIndicator.synchronize() edPluginControlBackground3D.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) if edPluginControlBackground3D.dataOutput.imageBackground != []: xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[0].estimate self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have background3D_estimate from all images: has_background3D_estimate = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.background3D_estimate is None: has_background3D_estimate = False if has_background3D_estimate: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.background3D_estimate.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator]) edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict( creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format( self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join( os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath( os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath( os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath( os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format( data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')