def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginAccumulatorv1_0.preProcess") with self.__class__._semaphore: for oneXsdItem in self.dataInput.item: #could be empty self._items.append(oneXsdItem.value) for oneXsdQuery in self.dataInput.query: #could be an empty list query = [] for oneXsdItem in oneXsdQuery.item: query.append(oneXsdItem.value) if oneXsdQuery.removeItems is not None: if oneXsdQuery.removeItems.value in [1, True, "true"]: removeItems = True elif oneXsdQuery.removeItems.value in [ 0, False, "false", None ]: removeItems = False else: removeItems = True else: removeItems = True query.sort() self._queries[tuple(query)] = removeItems if self.dataInput.flush is not None: if self.dataInput.flush.value in [1, True, "true"]: self.flush = True
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginAccumulatorv1_0.preProcess") self.__semaphore.acquire() for oneXsdItem in self.getDataInput().getItem(): #could be empty self.__items.append(oneXsdItem.getValue()) for oneXsdQuery in self.getDataInput().getQuery(): #could be an empty list query = [] for oneXsdItem in oneXsdQuery.getItem(): query.append(oneXsdItem.getValue()) if oneXsdQuery.getRemoveItems() is not None: if oneXsdQuery.getRemoveItems().getValue() in [1, True, "true"]: removeItems = True elif oneXsdQuery.getRemoveItems().getValue() in [0, False, "false", None]: removeItems = False else: removeItems = True else: removeItems = True query.sort() self.__queries[tuple(query)] = removeItems self.__semaphore.release() if self.getDataInput().getFlush() is not None: if self.getDataInput().getFlush().getValue() in [1, True, "true"]: self.flush = True
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginAccumulatorv1_0.preProcess") with self.__class__._semaphore: for oneXsdItem in self.dataInput.item: #could be empty self._items.append(oneXsdItem.value) for oneXsdQuery in self.dataInput.query: #could be an empty list query = [] for oneXsdItem in oneXsdQuery.item: query.append(oneXsdItem.value) if oneXsdQuery.removeItems is not None: if oneXsdQuery.removeItems.value in [1, True, "true"]: removeItems = True elif oneXsdQuery.removeItems.value in [0, False, "false", None]: removeItems = False else: removeItems = True else: removeItems = True query.sort() self._queries[tuple(query)] = removeItems if self.dataInput.flush is not None: if self.dataInput.flush.value in [1, True, "true"]: self.flush = True
def preProcess(self, _edObject=None): EDPlugin.preProcess(self, _edObject) EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.preProcess") # Construct python script for executing the plugin strPythonWrapper = self.preparePythonWrapperScript() self.__strPathToPythonWrapper = os.path.join(self.getWorkingDirectory(), self.__strPythonWrapperScriptName) EDUtilsFile.writeFile(self.__strPathToPythonWrapper, strPythonWrapper)
def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataInputAccumulator) self.xsDataResult = XSDataResultAccumulator() self.flush = False
def finallyProcess(self, _edObject=None): """ after processing of the plugin: Release a CPU resource by releasing the semaphore """ self.DEBUG("Release semaphore nbCPU by plugin %s" % self.getPluginName()) EDUtilsParallel.semaphoreNbThreadsRelease() EDPlugin.finallyProcess(self, _edObject)
def __init__ (self): """ """ EDPlugin.__init__(self) self.__strPluginToBeControlledName = None self.__dictControlledPlugins = {} self.__edActionCluster = None self.__listOfLoadedPlugins = []
def __init__(self): """ """ EDPlugin.__init__(self) self.__strPluginToBeControlledName = None self.__dictControlledPlugins = {} self.__edActionCluster = None self.__listOfLoadedPlugins = []
def finallyProcess(self, _edObject=None): """ after processing of the plugin: Release a CPU resource by releasing the semaphore """ EDVerbose.DEBUG("Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue())) EDUtilsParallel.semaphoreNbThreadsRelease() EDPlugin.finallyProcess(self, _edObject)
def __init__(self): """ Constructor of the plugin """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataInputAccumulator) self.xsDataResult = XSDataResultAccumulator() self.flush = False
def preProcess(self, _edObject=None): """ preProcess of the plugin: Ensure a CPU resource is available for the processing by acquiring a semaphore """ EDVerbose.DEBUG("Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue())) EDUtilsParallel.semaphoreNbThreadsAcquire() EDPlugin.preProcess(self, _edObject)
def preProcess(self, _edObject=None): """ preProcess of the plugin: Ensure a CPU resource is available for the processing by acquiring a semaphore """ self.DEBUG("Acquire semaphore nbCPU by plugin %s" % self.getPluginName()) EDUtilsParallel.semaphoreNbThreadsAcquire() EDPlugin.preProcess(self, _edObject)
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginWaitFile.preProcess") self.__filename = self.getDataInput().getExpectedFile().getPath().getValue() self.__expectedSize = self.getDataInput().getExpectedSize().getValue() if self.getDataInput().getTimeOut(): self.__timeout = self.getDataInput().getTimeOut().getValue() self.setTimeOut(self.getDataInput().getTimeOut().getValue() + EDPluginWaitFile.EXTRA_TIME)
def preProcess(self, _edObject=None): """ preProcess of the plugin: Ensure a CPU resource is available for the processing by acquiring a semaphore """ EDVerbose.DEBUG( "Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue())) EDUtilsParallel.semaphoreNbThreadsAcquire() EDPlugin.preProcess(self, _edObject)
def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataInputWaitFile) self.__filename = None self.__exists = None self.__filesize = None self.__expectedSize = None self.__timeout = self.DEFAULT_TIMEOUT
def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataInputWaitFile) self.__filename = None self.__exists = None self.__filesize = None self.__expectedSize = None self.__timeout = self.DEFAULT_TIMEOUT
def finallyProcess(self, _edObject=None): """ after processing of the plugin: Release a CPU resource by releasing the semaphore """ EDVerbose.DEBUG( "Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue())) EDUtilsParallel.semaphoreNbThreadsRelease() EDPlugin.finallyProcess(self, _edObject)
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginWaitFile.preProcess") self.__filename = self.getDataInput().getExpectedFile().getPath( ).getValue() self.__expectedSize = self.getDataInput().getExpectedSize().getValue() if self.getDataInput().getTimeOut(): self.__timeout = self.getDataInput().getTimeOut().getValue() self.setTimeOut(self.getDataInput().getTimeOut().getValue() + EDPluginWaitFile.EXTRA_TIME)
def __init__(self, _strNameOfPlugin): EDPlugin.__init__(self) self.__strNameOfPlugin = _strNameOfPlugin EDFactoryPluginStatic.loadModule(_strNameOfPlugin) self.__dictXMLDataInput = {} self.__dictXMLDataOutput = {} self.__strPythonWrapperScriptName = "pluginWrapperForJobScheduler.py" self.__strPathToPythonWrapper = None self.__edServerXMLRCP = EDServerXMLRCP.getInstance() self.__edServerXMLRCP.registerPlugin(self) self.__bFinished = False
def postProcess(self, _edObject=None): EDPlugin.postProcess(self) self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" % self.getRunTime()) xsDataResult = XSDataResultWaitFile() if os.path.exists(self.__filename): xsDataFile = XSDataFile() xsDataFile.setPath(XSDataString(self.__filename)) xsDataResult.setActualFile(xsDataFile) xsDataResult.setActualSize(XSDataInteger(os.path.getsize(self.__filename))) xsDataResult.setTimedOut(XSDataBoolean(self.getRunTime() >= self.__timeout)) # Create some output data self.setDataOutput(xsDataResult)
def postProcess(self, _edObject=None): EDPlugin.postProcess(self) self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" % self.getRunTime()) xsDataResult = XSDataResultWaitFile() if os.path.exists(self.__filename): xsDataFile = XSDataFile() xsDataFile.setPath(XSDataString(self.__filename)) xsDataResult.setActualFile(xsDataFile) xsDataResult.setActualSize( XSDataInteger(os.path.getsize(self.__filename))) xsDataResult.setTimedOut( XSDataBoolean(self.getRunTime() >= self.__timeout)) # Create some output data self.setDataOutput(xsDataResult)
def process(self, _edObject=None): EDPlugin.process(self) output = XSDataXscaleParsedOutput() # get all the file's contents, find where the info is and then # use helper functions to retrieve stuff and put it in the # data model try: f = open(self.dataInput.lp_file.value, 'r') lines = f.readlines() except IOError: strErrorMessage = "Input file {0} could not be opened".format( self.dataInput.lp_file.value) self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() return # look for the stats table info_begin = None info_end = None started = False for line_no, line in enumerate(lines): if line.find( 'LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected' ) != -1: # there's an empty line after the header info_begin = line_no + 2 if info_begin is not None and line.find('total') != -1: # we're at the last table line info_end = line_no # stop here as there are some other lines containing # "total" later break if info_begin is None or info_end is None: strErrorMessage = "Could not find the completeness table" self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() return _extract_completeness_entries(lines[info_begin:info_end + 1], output) input_file = self.dataInput.lp_file.value output.xds_run_directory = XSDataString(os.path.dirname(input_file)) self.dataOutput = output
def process(self, _edObject=None): EDPlugin.process(self) self.DEBUG("EDPluginWaitFile.process") self.DEBUG("EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s" % (self.getTimeOut(), self.__timeout)) self.setTimeInit() dirname = os.path.dirname(self.__filename) while self.getRunTime() < self.__timeout: if os.path.isdir(dirname): fd = os.open(dirname, os.O_RDONLY) os.fstat(fd) os.close(fd) if os.path.exists(self.__filename): self.__filesize = os.path.getsize(self.__filename) if self.__filesize >= self.__expectedSize: break time.sleep(EDPluginWaitFile.DELTA_TIME) self.setTimeEnd()
def testCreateBaseName(self): # Test 1 : naming of working directory edPlugin = EDPlugin() edPlugin.setBaseName("EDPlugin_testCreateBaseName") edPlugin.configure() strWorkingDir = edPlugin.getWorkingDirectory() EDAssert.equal(True, os.path.exists(strWorkingDir), "Test 1 : naming of working directory")
def configure(self): """ Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes. """ EDPlugin.configure(self) EDVerbose.DEBUG("EDPluginControl.configure") strControlledPlugins = self.config.get("controlledPlugins", None) if (strControlledPlugins != None): pyListControlledPlugins = strControlledPlugins.split(",") for strControlledPlugin in pyListControlledPlugins: strControlledPluginName = self.getStringConfigurationParameterValue(strControlledPlugin) if strControlledPluginName != None: self.setControlledPluginName(strControlledPlugin, strControlledPluginName) EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName)) clusterSize = self.config.get("clusterSize", None) if (clusterSize != None): self.__iClusterSize = int(strClusterSize) EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def process(self, _edObject=None): EDPlugin.process(self) output = XSDataXscaleParsedOutput() # get all the file's contents, find where the info is and then # use helper functions to retrieve stuff and put it in the # data model try: f = open(self.dataInput.lp_file.value, 'r') lines = f.readlines() except IOError: strErrorMessage = "Input file {0} could not be opened".format(self.dataInput.lp_file.value) self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() return # look for the stats table info_begin = None info_end = None started = False for line_no, line in enumerate(lines): if line.find('LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected') != -1: # there's an empty line after the header info_begin = line_no + 2 if info_begin is not None and line.find('total') != -1: # we're at the last table line info_end = line_no # stop here as there are some other lines containing # "total" later break if info_begin is None or info_end is None: strErrorMessage = "Could not find the completeness table" self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure() return _extract_completeness_entries(lines[info_begin:info_end + 1], output) input_file = self.dataInput.lp_file.value output.xds_run_directory = XSDataString(os.path.dirname(input_file)) self.dataOutput = output
def process(self, _edObject=None): EDPlugin.process(self) self.DEBUG("EDPluginWaitFile.process") self.DEBUG( "EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s" % (self.getTimeOut(), self.__timeout)) self.setTimeInit() dirname = os.path.dirname(self.__filename) while self.getRunTime() < self.__timeout: if os.path.isdir(dirname): fd = os.open(dirname, os.O_RDONLY) os.fstat(fd) os.close(fd) if os.path.exists(self.__filename): self.__filesize = os.path.getsize(self.__filename) if self.__filesize >= self.__expectedSize: break time.sleep(EDPluginWaitFile.DELTA_TIME) self.setTimeEnd()
def testCreateBaseName(self): # Test 1 : naming of working directory edPlugin = EDPlugin() edPlugin.setBaseName("EDPlugin_testCreateBaseName") edPlugin.configure() strWorkingDir = edPlugin.getWorkingDirectory() EDAssert.equal(True, os.path.exists(strWorkingDir), "Test 1 : naming of working directory")
def configure(self): """ Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes. """ EDPlugin.configure(self) pluginConfiguration = self.getConfiguration() if(pluginConfiguration == None): EDVerbose.DEBUG("EDPluginControl.configure: pluginConfiguration is None, using default settings") else: EDVerbose.DEBUG("EDPluginControl.configure: pluginConfiguration found, using settings from there") strControlledPlugins = EDConfiguration.getStringParamValue(pluginConfiguration, "controlledPlugins") if (strControlledPlugins != None): pyListControlledPlugins = strControlledPlugins.split(",") for strControlledPlugin in pyListControlledPlugins: strControlledPluginName = EDConfiguration.getStringParamValue(pluginConfiguration, strControlledPlugin) if strControlledPluginName != None: self.setControlledPluginName(strControlledPlugin, strControlledPluginName) EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName)) strClusterSize = EDConfiguration.getStringParamValue(pluginConfiguration, "clusterSize") if (strClusterSize != None): self.__iClusterSize = int(strClusterSize) EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def process(self, _edObject=None): EDPlugin.process(self) self.DEBUG("EDPluginAccumulatorv1_0.process") queriesToRemove = [] listXsdQuery = [] if not self.flush: self.__semaphore.acquire() for query in self.__queries: present = True for item in query: if not item in self.__items: present = False break if present is True: queriesToRemove.append(query) xsdQuery = XSDataQuery() xsdQuery.setItem([XSDataString(item) for item in query]) if self.__queries[query] is True: xsdQuery.setRemoveItems(XSDataBoolean(True)) for item in query: self.__items.remove(item) else: xsdQuery.setRemoveItems(XSDataBoolean(False)) listXsdQuery.append(xsdQuery) #Remove the query from the list of queries for query in queriesToRemove: self.__queries.pop(query) self.__semaphore.release() else: xsdQuery = XSDataQuery() self.__semaphore.acquire() xsdQuery.setItem([XSDataString(item) for item in self.__items]) xsdQuery.setRemoveItems(XSDataBoolean(True)) self.__class__.__items = [] self.__semaphore.release() listXsdQuery.append(xsdQuery) self.xsDataResult.setQuery(listXsdQuery)
def configure(self): """ Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes. """ EDPlugin.configure(self) self.DEBUG("EDPluginControl.configure") strControlledPlugins = self.config.get("controlledPlugins", None) if strControlledPlugins is not None: pyListControlledPlugins = strControlledPlugins.split(",") for strControlledPlugin in pyListControlledPlugins: strControlledPluginName = self.config.get(strControlledPlugin) if strControlledPluginName != None: self.setControlledPluginName(strControlledPlugin, strControlledPluginName) self.DEBUG( "EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName)) strClusterSize = self.config.get("clusterSize", None) if strClusterSize is not None: self.__iClusterSize = int(strClusterSize) self.DEBUG( "EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def process(self, _edObject=None): EDPlugin.process(self) self.DEBUG("EDPluginAccumulatorv1_0.process") queriesToRemove = [] listXsdQuery = [] if not self.flush: with self.__class__._semaphore: for query in self._queries: present = True for item in query: if not item in self._items: present = False break if present is True: queriesToRemove.append(query) xsdQuery = XSDataQuery() xsdQuery.setItem( [XSDataString(item) for item in query]) if self._queries[query] is True: xsdQuery.setRemoveItems(XSDataBoolean(True)) for item in query: self._items.remove(item) else: xsdQuery.setRemoveItems(XSDataBoolean(False)) listXsdQuery.append(xsdQuery) #Remove the query from the list of queries for query in queriesToRemove: self._queries.pop(query) else: xsdQuery = XSDataQuery() with self.__class__._semaphore: xsdQuery.setItem([XSDataString(item) for item in self._items]) xsdQuery.setRemoveItems(XSDataBoolean(True)) self.__class__._items = [] listXsdQuery.append(xsdQuery) self.xsDataResult.setQuery(listXsdQuery)
def __init__(self): EDPlugin.__init__(self) self.setXSDataInputClass(XSDataXscaleParsingInput) self.setDataOutput(XSDataXscaleParsedOutput())
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginParseXscaleOutputv1_0.preProcess")
def process(self, _edObject = None): EDPlugin.process(self) completeness_cutoff_param = self.dataInput.completeness_cutoff if completeness_cutoff_param is None: completeness_cutoff = 80 else: completeness_cutoff = completeness_cutoff_param.value isig_cutoff_param = self.dataInput.isig_cutoff if isig_cutoff_param is None: isig_cutoff = 3 else: isig_cutoff = isig_cutoff_param.value res_override = self.dataInput.res_override bins = list() # for the first iteration # comment from max's code: "less stringent at low res" local_completeness_cutoff = 70 # declared but not initialized in the perl code prev_isig = prev_res = 0 # XXX: if res is still not defined at the end it is set to # detector_max_res, which we should somehow defined (in the # data model?) and used as the default value before we start # the processing for entry in self.dataInput.completeness_entries: outer_res = entry.outer_res.value outer_complete = entry.outer_complete.value outer_rfactor = entry.outer_rfactor.value outer_isig = entry.outer_isig.value if outer_complete < local_completeness_cutoff or outer_isig < isig_cutoff or \ (res_override is not None and outer_res < res_override.value): if outer_complete < completeness_cutoff: EDVerbose.DEBUG('incomplete data (%s) in this shell' % outer_complete) res = prev_res else: res = _calculate_res_from_bins(prev_isig, prev_res, outer_isig, outer_res) bins.append(outer_res) #NOTE: get out of the loop, see the value of `skip` in #max's code break else: bins.append(outer_res) prev_res, prev_isig = outer_res, outer_isig # Now the implementation of what max does when he encouters # the total values, which are conveniently already parsed in # our case if len(bins) < 2: EDVerbose.DEBUG("No bins with I/sigma greater than %s" % isig_cutoff) EDVerbose.DEBUG("""something could be wrong, or the completeness could be too low! bravais lattice/SG could be incorrect or something more insidious like incorrect parameters in XDS.INP like distance, X beam, Y beam, etc. Stopping""") self.setFailure() return if res_override is not None: res = res_override.value # remove last bin (see why w/ max) retbins = [XSDataFloat(x) for x in bins[:-1]] data_output = XSDataResCutoffResult() data_output.res = XSDataFloat(res) data_output.bins = retbins totals = self.dataInput.total_completeness data_output.total_complete = totals.outer_complete data_output.total_rfactor = totals.outer_rfactor data_output.total_isig = totals.outer_isig self.dataOutput = data_output
def process(self, _edObject=None): EDPlugin.process(self) detector_max_res = self.dataInput.detector_max_res if detector_max_res is not None: detector_max_res = detector_max_res.value completeness_cutoff_param = self.dataInput.completeness_cutoff if completeness_cutoff_param is None: completeness_cutoff = 80 else: completeness_cutoff = completeness_cutoff_param.value isig_cutoff_param = self.dataInput.isig_cutoff if isig_cutoff_param is None: isig_cutoff = 3 else: isig_cutoff = isig_cutoff_param.value res_override = self.dataInput.res_override bins = list() # for the first iteration # comment from max's code: "less stringent at low res" local_completeness_cutoff = 70 # declared but not initialized in the perl code prev_isig = prev_res = 0 # XXX: if res is still not defined at the end it is set to # detector_max_res, which we should somehow defined (in the # data model?) and used as the default value before we start # the processing res = detector_max_res for entry in self.dataInput.completeness_entries: outer_res = entry.outer_res.value outer_complete = entry.outer_complete.value outer_rfactor = entry.outer_rfactor.value outer_isig = entry.outer_isig.value if outer_complete < local_completeness_cutoff or outer_isig < isig_cutoff or \ (res_override is not None and outer_res < res_override.value): if outer_complete < completeness_cutoff: EDVerbose.DEBUG('incomplete data (%s) in this shell' % outer_complete) res = prev_res else: res = _calculate_res_from_bins(prev_isig, prev_res, outer_isig, outer_res, isig_cutoff) bins.append(outer_res) #NOTE: get out of the loop, see the value of `skip` in #max's code break else: bins.append(outer_res) prev_res, prev_isig = outer_res, outer_isig # Now the implementation of what max does when he encouters # the total values, which are conveniently already parsed in # our case if len(bins) < 2: EDVerbose.DEBUG("No bins with I/sigma greater than %s" % isig_cutoff) EDVerbose.DEBUG( """something could be wrong, or the completeness could be too low! bravais lattice/SG could be incorrect or something more insidious like incorrect parameters in XDS.INP like distance, X beam, Y beam, etc. Stopping""") self.setFailure() return if res is None: res = sorted(bins)[0] if res_override is not None: res = res_override.value # remove last bin (see why w/ max) retbins = [XSDataFloat(x) for x in bins[:-1]] data_output = XSDataResCutoffResult() data_output.res = XSDataFloat(res) data_output.bins = retbins totals = self.dataInput.total_completeness data_output.total_complete = totals.outer_complete data_output.total_rfactor = totals.outer_rfactor data_output.total_isig = totals.outer_isig self.dataOutput = data_output
def postProcess(self, _edObject=None): EDPlugin.postProcess(self) EDVerbose.DEBUG("*** EDPluginExecReadImageHeaderADSCv10.postProcess") if (self.__xsDataResultReadImageHeader is not None): self.setDataOutput(self.__xsDataResultReadImageHeader)
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginResCutoffv1_0.preProcess")
def postProcess(self, _edObject = None): EDPlugin.postProcess(self) self.DEBUG("EDPluginParseXdsOutput.postProcess")
def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginParseXscaleOutputv1_0.preProcess")
def __init__(self): EDPlugin.__init__(self) self.setXSDataInputClass(XSDataXscaleParsingInput) self.setDataOutput(XSDataXscaleParsedOutput())
def finallyProcess(self, _edObject=None): EDPlugin.finallyProcess(self, _edObject) EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.finallyProcess") while not self.__bFinished: time.sleep(1)
def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataResCutoff)
def __init__( self ): EDPlugin.__init__( self ) self.setXSDataInputClass( XSDataString ) self.setXSDataInputClass( XSDataString, "value1" ) self.setXSDataInputClass( XSDataString, "value2" )
def postProcess(self, _edObject=None): EDPlugin.postProcess(self) self.DEBUG("EDPluginParseXdsOutput.postProcess")
def __init__(self): EDPlugin.__init__(self) self.setXSDataInputClass(XSDataResCutoff) self.setDataOutput(XSDataResCutoffResult())
def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataXdsOutputFile)
def process(self, _edObject=None): EDPlugin.process(self) detector_max_res = self.dataInput.detector_max_res if detector_max_res is not None: detector_max_res = detector_max_res.value completeness_cutoff_param = self.dataInput.completeness_cutoff if completeness_cutoff_param is None: completeness_cutoff = 80 else: completeness_cutoff = completeness_cutoff_param.value isig_cutoff_param = self.dataInput.isig_cutoff if isig_cutoff_param is None: isig_cutoff = 3 else: isig_cutoff = isig_cutoff_param.value cc_half_cutoff_param = self.dataInput.cc_half_cutoff if cc_half_cutoff_param is not None: cc_half_cutoff = cc_half_cutoff_param.value else: cc_half_cutoff = 30 res_override = self.dataInput.res_override bins = list() # for the first iteration # comment from max's code: "less stringent at low res" local_completeness_cutoff = 70 # declared but not initialized in the perl code prev_isig = prev_res = 0 # XXX: if res is still not defined at the end it is set to # detector_max_res, which we should somehow defined (in the # data model?) and used as the default value before we start # the processing res = detector_max_res for entry in self.dataInput.completeness_entries: current_res = entry.res.value complete = entry.complete.value rfactor = entry.rfactor.value isig = entry.isig.value cc_half = entry.half_dataset_correlation.value # isig < isig_cutoff or \ if cc_half < cc_half_cutoff or \ (res_override is not None and current_res < res_override.value): continue else: bins.append(current_res) # Now the implementation of what max does when he encouters # the total values, which are conveniently already parsed in # our case if len(bins) < 2: strErrorMessage = "No bins with CC1/2 greater than %s" % cc_half_cutoff self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) self.ERROR( "Something could be wrong, or the completeness could be too low!" ) self.ERROR( "bravais lattice/SG could be incorrect or something more insidious like" ) self.ERROR( "incorrect parameters in XDS.INP like distance, X beam, Y beam, etc." ) self.ERROR("Stopping") self.setFailure() return if res is None: res = sorted(bins)[0] if res_override is not None: res = res_override.value retbins = [XSDataDouble(x) for x in bins] data_output = XSDataResCutoffResult() data_output.res = XSDataDouble(res) data_output.bins = retbins totals = self.dataInput.total_completeness data_output.total_complete = totals.complete data_output.total_rfactor = totals.rfactor data_output.total_isig = totals.isig self.dataOutput = data_output
def __init__(self ): """ """ EDPlugin.__init__(self ) self.setXSDataInputClass(XSDataResCutoff)
def process(self, _edObject = None): EDPlugin.process(self) output = XSDataXdsOutput() # get all the file's contents, find where the info is and then # use helper functions to retrieve stuff and put it in the # data model try: f = open(self.dataInput.correct_lp.path.value, 'r') lines = f.readlines() except IOError: EDVerbose.ERROR('Could not open the specified XDS output file for reading') self.setFailure() return # look for the "big piece of information" info_begin = None info_end = None for lineno, line in enumerate(lines): if info_begin is None: if line.find('REFINEMENT OF DIFFRACTION PARAMETERS USING ALL IMAGES') != -1: info_begin = lineno else: if line.find('MEAN INTENSITY AS FUNCTION OF SPINDLE POSITION WITHIN DATA IMAGE') != -1: info_end = lineno break if info_begin is None or info_end is None: EDVerbose.ERROR('could not find the refined parameters') self.setFailure() return _extract_infos(lines[info_begin:info_end], output) # second pass, look for the interesting table info_begin = None info_end = None started = False for line_no, line in enumerate(lines): if line.find('REFLECTIONS OF TYPE H,0,0 0,K,0 0,0,L OR EXPECTED TO BE ABSENT (*)') != -1: # the table will start shortly after started = True continue if started: # look if we are at the table yet if line.find('LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected') != -1: # there's an empty line after the header info_begin = line_no + 2 if info_begin is not None and line.find('total') != -1: # we're at the last table line info_end = line_no if info_begin is None or info_end is None: EDVerbose.ERROR('could not find the completeness table') self.setFailure() return _extract_completeness_entries(lines[info_begin:info_end+1], output) # now for the last bit: check if we were given a path to the # gxparm file and if it exists get the space group and unit # cell constants from it if self.dataInput.gxparm is not None: gxparm_path = self.dataInput.gxparm.path.value if os.path.exists(gxparm_path): with open(gxparm_path, 'r') as f: lines = f.readlines() for line in lines: # the one we want has 7 floats chunks = line.split() if len(chunks) == 7: output.sg_number = XSDataInteger(int(chunks[0])) output.unit_cell_constants = [XSDataFloat(float(x)) for x in chunks[1:]] input_file = self.dataInput.correct_lp.path.value output.xds_run_directory = XSDataString(os.path.dirname(input_file)) self.dataOutput = output
def process(self, _edObject=None): EDPlugin.process(self, _edObject) EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.process") #print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" EDVerbose.DEBUG("Executing: oarsub \"python %s\"" % self.__strPathToPythonWrapper) os.system("oarsub -l \"{cpu_vendor='INTEL'}/core=1\" \"python %s\"" % self.__strPathToPythonWrapper)
def __init__(self ): """ """ EDPlugin.__init__(self ) self.setXSDataInputClass(XSDataXdsOutputFile)
def process(self, _edObject=None): EDPlugin.process(self) output = XSDataXdsOutput() # get all the file's contents, find where the info is and then # use helper functions to retrieve stuff and put it in the # data model try: f = open(self.dataInput.correct_lp.path.value, 'r') lines = f.readlines() except IOError: EDVerbose.ERROR( 'Could not open the specified XDS output file for reading') self.setFailure() return # look for the "big piece of information" info_begin = None info_end = None for lineno, line in enumerate(lines): if info_begin is None: if line.find( 'REFINEMENT OF DIFFRACTION PARAMETERS USING ALL IMAGES' ) != -1: info_begin = lineno else: if line.find( 'MEAN INTENSITY AS FUNCTION OF SPINDLE POSITION WITHIN DATA IMAGE' ) != -1: info_end = lineno break if info_begin is None or info_end is None: EDVerbose.ERROR('could not find the refined parameters') self.setFailure() return _extract_infos(lines[info_begin:info_end], output) # second pass, look for the interesting table info_begin = None info_end = None started = False for line_no, line in enumerate(lines): if line.find( 'REFLECTIONS OF TYPE H,0,0 0,K,0 0,0,L OR EXPECTED TO BE ABSENT (*)' ) != -1: # the table will start shortly after started = True continue if started: # look if we are at the table yet if line.find( 'LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected' ) != -1: # there's an empty line after the header info_begin = line_no + 2 if info_begin is not None and line.find('total') != -1: # we're at the last table line info_end = line_no if info_begin is None or info_end is None: EDVerbose.ERROR('could not find the completeness table') self.setFailure() return _extract_completeness_entries(lines[info_begin:info_end + 1], output) # now for the last bit: check if we were given a path to the # gxparm file and if it exists get the space group and unit # cell constants from it if self.dataInput.gxparm is not None: gxparm_path = self.dataInput.gxparm.path.value if os.path.exists(gxparm_path): with open(gxparm_path, 'r') as f: lines = f.readlines() for line in lines: # the one we want has 7 floats chunks = line.split() if len(chunks) == 7: output.sg_number = XSDataInteger(int(chunks[0])) output.unit_cell_constants = [ XSDataFloat(float(x)) for x in chunks[1:] ] input_file = self.dataInput.correct_lp.path.value output.xds_run_directory = XSDataString(os.path.dirname(input_file)) self.dataOutput = output