def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginAccumulatorv1_0.preProcess")
with self.__class__._semaphore:
for oneXsdItem in self.dataInput.item: #could be empty
self._items.append(oneXsdItem.value)
for oneXsdQuery in self.dataInput.query: #could be an empty list
query = []
for oneXsdItem in oneXsdQuery.item:
query.append(oneXsdItem.value)
if oneXsdQuery.removeItems is not None:
if oneXsdQuery.removeItems.value in [1, True, "true"]:
removeItems = True
elif oneXsdQuery.removeItems.value in [
0, False, "false", None
]:
removeItems = False
else:
removeItems = True
else:
removeItems = True
query.sort()
self._queries[tuple(query)] = removeItems
if self.dataInput.flush is not None:
if self.dataInput.flush.value in [1, True, "true"]:
self.flush = True
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginAccumulatorv1_0.preProcess")
self.__semaphore.acquire()
for oneXsdItem in self.getDataInput().getItem(): #could be empty
self.__items.append(oneXsdItem.getValue())
for oneXsdQuery in self.getDataInput().getQuery(): #could be an empty list
query = []
for oneXsdItem in oneXsdQuery.getItem():
query.append(oneXsdItem.getValue())
if oneXsdQuery.getRemoveItems() is not None:
if oneXsdQuery.getRemoveItems().getValue() in [1, True, "true"]:
removeItems = True
elif oneXsdQuery.getRemoveItems().getValue() in [0, False, "false", None]:
removeItems = False
else:
removeItems = True
else:
removeItems = True
query.sort()
self.__queries[tuple(query)] = removeItems
self.__semaphore.release()
if self.getDataInput().getFlush() is not None:
if self.getDataInput().getFlush().getValue() in [1, True, "true"]:
self.flush = True
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginAccumulatorv1_0.preProcess")
with self.__class__._semaphore:
for oneXsdItem in self.dataInput.item: #could be empty
self._items.append(oneXsdItem.value)
for oneXsdQuery in self.dataInput.query: #could be an empty list
query = []
for oneXsdItem in oneXsdQuery.item:
query.append(oneXsdItem.value)
if oneXsdQuery.removeItems is not None:
if oneXsdQuery.removeItems.value in [1, True, "true"]:
removeItems = True
elif oneXsdQuery.removeItems.value in [0, False, "false", None]:
removeItems = False
else:
removeItems = True
else:
removeItems = True
query.sort()
self._queries[tuple(query)] = removeItems
if self.dataInput.flush is not None:
if self.dataInput.flush.value in [1, True, "true"]:
self.flush = True
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self, _edObject)
EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.preProcess")
# Construct python script for executing the plugin
strPythonWrapper = self.preparePythonWrapperScript()
self.__strPathToPythonWrapper = os.path.join(self.getWorkingDirectory(), self.__strPythonWrapperScriptName)
EDUtilsFile.writeFile(self.__strPathToPythonWrapper, strPythonWrapper)
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataInputAccumulator)
self.xsDataResult = XSDataResultAccumulator()
self.flush = False
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
self.DEBUG("Release semaphore nbCPU by plugin %s" % self.getPluginName())
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def __init__ (self):
"""
"""
EDPlugin.__init__(self)
self.__strPluginToBeControlledName = None
self.__dictControlledPlugins = {}
self.__edActionCluster = None
self.__listOfLoadedPlugins = []
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.__strPluginToBeControlledName = None
self.__dictControlledPlugins = {}
self.__edActionCluster = None
self.__listOfLoadedPlugins = []
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
EDVerbose.DEBUG("Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def __init__(self):
"""
Constructor of the plugin
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataInputAccumulator)
self.xsDataResult = XSDataResultAccumulator()
self.flush = False
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
EDVerbose.DEBUG("Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
self.DEBUG("Acquire semaphore nbCPU by plugin %s" % self.getPluginName())
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginWaitFile.preProcess")
self.__filename = self.getDataInput().getExpectedFile().getPath().getValue()
self.__expectedSize = self.getDataInput().getExpectedSize().getValue()
if self.getDataInput().getTimeOut():
self.__timeout = self.getDataInput().getTimeOut().getValue()
self.setTimeOut(self.getDataInput().getTimeOut().getValue() + EDPluginWaitFile.EXTRA_TIME)
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
EDVerbose.DEBUG(
"Acquire semaphore nbCPU by plugin %s, currently value: %s" %
(self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataInputWaitFile)
self.__filename = None
self.__exists = None
self.__filesize = None
self.__expectedSize = None
self.__timeout = self.DEFAULT_TIMEOUT
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataInputWaitFile)
self.__filename = None
self.__exists = None
self.__filesize = None
self.__expectedSize = None
self.__timeout = self.DEFAULT_TIMEOUT
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
EDVerbose.DEBUG(
"Release semaphore nbCPU by plugin %s, currently value: %s" %
(self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginWaitFile.preProcess")
self.__filename = self.getDataInput().getExpectedFile().getPath(
).getValue()
self.__expectedSize = self.getDataInput().getExpectedSize().getValue()
if self.getDataInput().getTimeOut():
self.__timeout = self.getDataInput().getTimeOut().getValue()
self.setTimeOut(self.getDataInput().getTimeOut().getValue() +
EDPluginWaitFile.EXTRA_TIME)
def __init__(self, _strNameOfPlugin):
EDPlugin.__init__(self)
self.__strNameOfPlugin = _strNameOfPlugin
EDFactoryPluginStatic.loadModule(_strNameOfPlugin)
self.__dictXMLDataInput = {}
self.__dictXMLDataOutput = {}
self.__strPythonWrapperScriptName = "pluginWrapperForJobScheduler.py"
self.__strPathToPythonWrapper = None
self.__edServerXMLRCP = EDServerXMLRCP.getInstance()
self.__edServerXMLRCP.registerPlugin(self)
self.__bFinished = False
def postProcess(self, _edObject=None):
EDPlugin.postProcess(self)
self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" % self.getRunTime())
xsDataResult = XSDataResultWaitFile()
if os.path.exists(self.__filename):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__filename))
xsDataResult.setActualFile(xsDataFile)
xsDataResult.setActualSize(XSDataInteger(os.path.getsize(self.__filename)))
xsDataResult.setTimedOut(XSDataBoolean(self.getRunTime() >= self.__timeout))
# Create some output data
self.setDataOutput(xsDataResult)
def postProcess(self, _edObject=None):
EDPlugin.postProcess(self)
self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" %
self.getRunTime())
xsDataResult = XSDataResultWaitFile()
if os.path.exists(self.__filename):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__filename))
xsDataResult.setActualFile(xsDataFile)
xsDataResult.setActualSize(
XSDataInteger(os.path.getsize(self.__filename)))
xsDataResult.setTimedOut(
XSDataBoolean(self.getRunTime() >= self.__timeout))
# Create some output data
self.setDataOutput(xsDataResult)
def process(self, _edObject=None):
EDPlugin.process(self)
output = XSDataXscaleParsedOutput()
# get all the file's contents, find where the info is and then
# use helper functions to retrieve stuff and put it in the
# data model
try:
f = open(self.dataInput.lp_file.value, 'r')
lines = f.readlines()
except IOError:
strErrorMessage = "Input file {0} could not be opened".format(
self.dataInput.lp_file.value)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# look for the stats table
info_begin = None
info_end = None
started = False
for line_no, line in enumerate(lines):
if line.find(
'LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected'
) != -1:
# there's an empty line after the header
info_begin = line_no + 2
if info_begin is not None and line.find('total') != -1:
# we're at the last table line
info_end = line_no
# stop here as there are some other lines containing
# "total" later
break
if info_begin is None or info_end is None:
strErrorMessage = "Could not find the completeness table"
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
_extract_completeness_entries(lines[info_begin:info_end + 1], output)
input_file = self.dataInput.lp_file.value
output.xds_run_directory = XSDataString(os.path.dirname(input_file))
self.dataOutput = output
def process(self, _edObject=None):
EDPlugin.process(self)
self.DEBUG("EDPluginWaitFile.process")
self.DEBUG("EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s" % (self.getTimeOut(), self.__timeout))
self.setTimeInit()
dirname = os.path.dirname(self.__filename)
while self.getRunTime() < self.__timeout:
if os.path.isdir(dirname):
fd = os.open(dirname, os.O_RDONLY)
os.fstat(fd)
os.close(fd)
if os.path.exists(self.__filename):
self.__filesize = os.path.getsize(self.__filename)
if self.__filesize >= self.__expectedSize:
break
time.sleep(EDPluginWaitFile.DELTA_TIME)
self.setTimeEnd()
def testCreateBaseName(self):
# Test 1 : naming of working directory
edPlugin = EDPlugin()
edPlugin.setBaseName("EDPlugin_testCreateBaseName")
edPlugin.configure()
strWorkingDir = edPlugin.getWorkingDirectory()
EDAssert.equal(True, os.path.exists(strWorkingDir), "Test 1 : naming of working directory")
def configure(self):
"""
Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
"""
EDPlugin.configure(self)
EDVerbose.DEBUG("EDPluginControl.configure")
strControlledPlugins = self.config.get("controlledPlugins", None)
if (strControlledPlugins != None):
pyListControlledPlugins = strControlledPlugins.split(",")
for strControlledPlugin in pyListControlledPlugins:
strControlledPluginName = self.getStringConfigurationParameterValue(strControlledPlugin)
if strControlledPluginName != None:
self.setControlledPluginName(strControlledPlugin, strControlledPluginName)
EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName))
clusterSize = self.config.get("clusterSize", None)
if (clusterSize != None):
self.__iClusterSize = int(strClusterSize)
EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def process(self, _edObject=None):
EDPlugin.process(self)
output = XSDataXscaleParsedOutput()
# get all the file's contents, find where the info is and then
# use helper functions to retrieve stuff and put it in the
# data model
try:
f = open(self.dataInput.lp_file.value, 'r')
lines = f.readlines()
except IOError:
strErrorMessage = "Input file {0} could not be opened".format(self.dataInput.lp_file.value)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# look for the stats table
info_begin = None
info_end = None
started = False
for line_no, line in enumerate(lines):
if line.find('LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected') != -1:
# there's an empty line after the header
info_begin = line_no + 2
if info_begin is not None and line.find('total') != -1:
# we're at the last table line
info_end = line_no
# stop here as there are some other lines containing
# "total" later
break
if info_begin is None or info_end is None:
strErrorMessage = "Could not find the completeness table"
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
_extract_completeness_entries(lines[info_begin:info_end + 1], output)
input_file = self.dataInput.lp_file.value
output.xds_run_directory = XSDataString(os.path.dirname(input_file))
self.dataOutput = output
def process(self, _edObject=None):
EDPlugin.process(self)
self.DEBUG("EDPluginWaitFile.process")
self.DEBUG(
"EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s"
% (self.getTimeOut(), self.__timeout))
self.setTimeInit()
dirname = os.path.dirname(self.__filename)
while self.getRunTime() < self.__timeout:
if os.path.isdir(dirname):
fd = os.open(dirname, os.O_RDONLY)
os.fstat(fd)
os.close(fd)
if os.path.exists(self.__filename):
self.__filesize = os.path.getsize(self.__filename)
if self.__filesize >= self.__expectedSize:
break
time.sleep(EDPluginWaitFile.DELTA_TIME)
self.setTimeEnd()
def testCreateBaseName(self):
# Test 1 : naming of working directory
edPlugin = EDPlugin()
edPlugin.setBaseName("EDPlugin_testCreateBaseName")
edPlugin.configure()
strWorkingDir = edPlugin.getWorkingDirectory()
EDAssert.equal(True, os.path.exists(strWorkingDir), "Test 1 : naming of working directory")
def configure(self):
"""
Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
"""
EDPlugin.configure(self)
pluginConfiguration = self.getConfiguration()
if(pluginConfiguration == None):
EDVerbose.DEBUG("EDPluginControl.configure: pluginConfiguration is None, using default settings")
else:
EDVerbose.DEBUG("EDPluginControl.configure: pluginConfiguration found, using settings from there")
strControlledPlugins = EDConfiguration.getStringParamValue(pluginConfiguration, "controlledPlugins")
if (strControlledPlugins != None):
pyListControlledPlugins = strControlledPlugins.split(",")
for strControlledPlugin in pyListControlledPlugins:
strControlledPluginName = EDConfiguration.getStringParamValue(pluginConfiguration, strControlledPlugin)
if strControlledPluginName != None:
self.setControlledPluginName(strControlledPlugin, strControlledPluginName)
EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName))
strClusterSize = EDConfiguration.getStringParamValue(pluginConfiguration, "clusterSize")
if (strClusterSize != None):
self.__iClusterSize = int(strClusterSize)
EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def process(self, _edObject=None):
EDPlugin.process(self)
self.DEBUG("EDPluginAccumulatorv1_0.process")
queriesToRemove = []
listXsdQuery = []
if not self.flush:
self.__semaphore.acquire()
for query in self.__queries:
present = True
for item in query:
if not item in self.__items:
present = False
break
if present is True:
queriesToRemove.append(query)
xsdQuery = XSDataQuery()
xsdQuery.setItem([XSDataString(item) for item in query])
if self.__queries[query] is True:
xsdQuery.setRemoveItems(XSDataBoolean(True))
for item in query:
self.__items.remove(item)
else:
xsdQuery.setRemoveItems(XSDataBoolean(False))
listXsdQuery.append(xsdQuery)
#Remove the query from the list of queries
for query in queriesToRemove:
self.__queries.pop(query)
self.__semaphore.release()
else:
xsdQuery = XSDataQuery()
self.__semaphore.acquire()
xsdQuery.setItem([XSDataString(item) for item in self.__items])
xsdQuery.setRemoveItems(XSDataBoolean(True))
self.__class__.__items = []
self.__semaphore.release()
listXsdQuery.append(xsdQuery)
self.xsDataResult.setQuery(listXsdQuery)
def configure(self):
"""
Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
"""
EDPlugin.configure(self)
self.DEBUG("EDPluginControl.configure")
strControlledPlugins = self.config.get("controlledPlugins", None)
if strControlledPlugins is not None:
pyListControlledPlugins = strControlledPlugins.split(",")
for strControlledPlugin in pyListControlledPlugins:
strControlledPluginName = self.config.get(strControlledPlugin)
if strControlledPluginName != None:
self.setControlledPluginName(strControlledPlugin,
strControlledPluginName)
self.DEBUG(
"EDPluginControl.configure: setting controlled plugin %s to specific plugin %s"
% (strControlledPlugin, strControlledPluginName))
strClusterSize = self.config.get("clusterSize", None)
if strClusterSize is not None:
self.__iClusterSize = int(strClusterSize)
self.DEBUG(
"EDPluginControl.configure: setting cluster size to %d" %
self.__iClusterSize)
def process(self, _edObject=None):
EDPlugin.process(self)
self.DEBUG("EDPluginAccumulatorv1_0.process")
queriesToRemove = []
listXsdQuery = []
if not self.flush:
with self.__class__._semaphore:
for query in self._queries:
present = True
for item in query:
if not item in self._items:
present = False
break
if present is True:
queriesToRemove.append(query)
xsdQuery = XSDataQuery()
xsdQuery.setItem(
[XSDataString(item) for item in query])
if self._queries[query] is True:
xsdQuery.setRemoveItems(XSDataBoolean(True))
for item in query:
self._items.remove(item)
else:
xsdQuery.setRemoveItems(XSDataBoolean(False))
listXsdQuery.append(xsdQuery)
#Remove the query from the list of queries
for query in queriesToRemove:
self._queries.pop(query)
else:
xsdQuery = XSDataQuery()
with self.__class__._semaphore:
xsdQuery.setItem([XSDataString(item) for item in self._items])
xsdQuery.setRemoveItems(XSDataBoolean(True))
self.__class__._items = []
listXsdQuery.append(xsdQuery)
self.xsDataResult.setQuery(listXsdQuery)
def __init__(self):
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataXscaleParsingInput)
self.setDataOutput(XSDataXscaleParsedOutput())
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginParseXscaleOutputv1_0.preProcess")
def process(self, _edObject = None):
EDPlugin.process(self)
completeness_cutoff_param = self.dataInput.completeness_cutoff
if completeness_cutoff_param is None:
completeness_cutoff = 80
else:
completeness_cutoff = completeness_cutoff_param.value
isig_cutoff_param = self.dataInput.isig_cutoff
if isig_cutoff_param is None:
isig_cutoff = 3
else:
isig_cutoff = isig_cutoff_param.value
res_override = self.dataInput.res_override
bins = list()
# for the first iteration
# comment from max's code: "less stringent at low res"
local_completeness_cutoff = 70
# declared but not initialized in the perl code
prev_isig = prev_res = 0
# XXX: if res is still not defined at the end it is set to
# detector_max_res, which we should somehow defined (in the
# data model?) and used as the default value before we start
# the processing
for entry in self.dataInput.completeness_entries:
outer_res = entry.outer_res.value
outer_complete = entry.outer_complete.value
outer_rfactor = entry.outer_rfactor.value
outer_isig = entry.outer_isig.value
if outer_complete < local_completeness_cutoff or outer_isig < isig_cutoff or \
(res_override is not None and outer_res < res_override.value):
if outer_complete < completeness_cutoff:
EDVerbose.DEBUG('incomplete data (%s) in this shell' % outer_complete)
res = prev_res
else:
res = _calculate_res_from_bins(prev_isig, prev_res,
outer_isig, outer_res)
bins.append(outer_res)
#NOTE: get out of the loop, see the value of `skip` in
#max's code
break
else:
bins.append(outer_res)
prev_res, prev_isig = outer_res, outer_isig
# Now the implementation of what max does when he encouters
# the total values, which are conveniently already parsed in
# our case
if len(bins) < 2:
EDVerbose.DEBUG("No bins with I/sigma greater than %s" % isig_cutoff)
EDVerbose.DEBUG("""something could be wrong, or the completeness could be too low!
bravais lattice/SG could be incorrect or something more insidious like
incorrect parameters in XDS.INP like distance, X beam, Y beam, etc.
Stopping""")
self.setFailure()
return
if res_override is not None:
res = res_override.value
# remove last bin (see why w/ max)
retbins = [XSDataFloat(x) for x in bins[:-1]]
data_output = XSDataResCutoffResult()
data_output.res = XSDataFloat(res)
data_output.bins = retbins
totals = self.dataInput.total_completeness
data_output.total_complete = totals.outer_complete
data_output.total_rfactor = totals.outer_rfactor
data_output.total_isig = totals.outer_isig
self.dataOutput = data_output
def process(self, _edObject=None):
EDPlugin.process(self)
detector_max_res = self.dataInput.detector_max_res
if detector_max_res is not None:
detector_max_res = detector_max_res.value
completeness_cutoff_param = self.dataInput.completeness_cutoff
if completeness_cutoff_param is None:
completeness_cutoff = 80
else:
completeness_cutoff = completeness_cutoff_param.value
isig_cutoff_param = self.dataInput.isig_cutoff
if isig_cutoff_param is None:
isig_cutoff = 3
else:
isig_cutoff = isig_cutoff_param.value
res_override = self.dataInput.res_override
bins = list()
# for the first iteration
# comment from max's code: "less stringent at low res"
local_completeness_cutoff = 70
# declared but not initialized in the perl code
prev_isig = prev_res = 0
# XXX: if res is still not defined at the end it is set to
# detector_max_res, which we should somehow defined (in the
# data model?) and used as the default value before we start
# the processing
res = detector_max_res
for entry in self.dataInput.completeness_entries:
outer_res = entry.outer_res.value
outer_complete = entry.outer_complete.value
outer_rfactor = entry.outer_rfactor.value
outer_isig = entry.outer_isig.value
if outer_complete < local_completeness_cutoff or outer_isig < isig_cutoff or \
(res_override is not None and outer_res < res_override.value):
if outer_complete < completeness_cutoff:
EDVerbose.DEBUG('incomplete data (%s) in this shell' %
outer_complete)
res = prev_res
else:
res = _calculate_res_from_bins(prev_isig, prev_res,
outer_isig, outer_res,
isig_cutoff)
bins.append(outer_res)
#NOTE: get out of the loop, see the value of `skip` in
#max's code
break
else:
bins.append(outer_res)
prev_res, prev_isig = outer_res, outer_isig
# Now the implementation of what max does when he encouters
# the total values, which are conveniently already parsed in
# our case
if len(bins) < 2:
EDVerbose.DEBUG("No bins with I/sigma greater than %s" %
isig_cutoff)
EDVerbose.DEBUG(
"""something could be wrong, or the completeness could be too low!
bravais lattice/SG could be incorrect or something more insidious like
incorrect parameters in XDS.INP like distance, X beam, Y beam, etc.
Stopping""")
self.setFailure()
return
if res is None:
res = sorted(bins)[0]
if res_override is not None:
res = res_override.value
# remove last bin (see why w/ max)
retbins = [XSDataFloat(x) for x in bins[:-1]]
data_output = XSDataResCutoffResult()
data_output.res = XSDataFloat(res)
data_output.bins = retbins
totals = self.dataInput.total_completeness
data_output.total_complete = totals.outer_complete
data_output.total_rfactor = totals.outer_rfactor
data_output.total_isig = totals.outer_isig
self.dataOutput = data_output
def postProcess(self, _edObject=None):
EDPlugin.postProcess(self)
EDVerbose.DEBUG("*** EDPluginExecReadImageHeaderADSCv10.postProcess")
if (self.__xsDataResultReadImageHeader is not None):
self.setDataOutput(self.__xsDataResultReadImageHeader)
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginResCutoffv1_0.preProcess")
def postProcess(self, _edObject = None):
EDPlugin.postProcess(self)
self.DEBUG("EDPluginParseXdsOutput.postProcess")
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginParseXscaleOutputv1_0.preProcess")
def __init__(self):
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataXscaleParsingInput)
self.setDataOutput(XSDataXscaleParsedOutput())
def finallyProcess(self, _edObject=None):
EDPlugin.finallyProcess(self, _edObject)
EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.finallyProcess")
while not self.__bFinished:
time.sleep(1)
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataResCutoff)
def __init__( self ):
EDPlugin.__init__( self )
self.setXSDataInputClass( XSDataString )
self.setXSDataInputClass( XSDataString, "value1" )
self.setXSDataInputClass( XSDataString, "value2" )
def postProcess(self, _edObject=None):
EDPlugin.postProcess(self)
self.DEBUG("EDPluginParseXdsOutput.postProcess")
def __init__(self):
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataResCutoff)
self.setDataOutput(XSDataResCutoffResult())
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataXdsOutputFile)
def process(self, _edObject=None):
EDPlugin.process(self)
detector_max_res = self.dataInput.detector_max_res
if detector_max_res is not None:
detector_max_res = detector_max_res.value
completeness_cutoff_param = self.dataInput.completeness_cutoff
if completeness_cutoff_param is None:
completeness_cutoff = 80
else:
completeness_cutoff = completeness_cutoff_param.value
isig_cutoff_param = self.dataInput.isig_cutoff
if isig_cutoff_param is None:
isig_cutoff = 3
else:
isig_cutoff = isig_cutoff_param.value
cc_half_cutoff_param = self.dataInput.cc_half_cutoff
if cc_half_cutoff_param is not None:
cc_half_cutoff = cc_half_cutoff_param.value
else:
cc_half_cutoff = 30
res_override = self.dataInput.res_override
bins = list()
# for the first iteration
# comment from max's code: "less stringent at low res"
local_completeness_cutoff = 70
# declared but not initialized in the perl code
prev_isig = prev_res = 0
# XXX: if res is still not defined at the end it is set to
# detector_max_res, which we should somehow defined (in the
# data model?) and used as the default value before we start
# the processing
res = detector_max_res
for entry in self.dataInput.completeness_entries:
current_res = entry.res.value
complete = entry.complete.value
rfactor = entry.rfactor.value
isig = entry.isig.value
cc_half = entry.half_dataset_correlation.value
# isig < isig_cutoff or \
if cc_half < cc_half_cutoff or \
(res_override is not None and current_res < res_override.value):
continue
else:
bins.append(current_res)
# Now the implementation of what max does when he encouters
# the total values, which are conveniently already parsed in
# our case
if len(bins) < 2:
strErrorMessage = "No bins with CC1/2 greater than %s" % cc_half_cutoff
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.ERROR(
"Something could be wrong, or the completeness could be too low!"
)
self.ERROR(
"bravais lattice/SG could be incorrect or something more insidious like"
)
self.ERROR(
"incorrect parameters in XDS.INP like distance, X beam, Y beam, etc."
)
self.ERROR("Stopping")
self.setFailure()
return
if res is None:
res = sorted(bins)[0]
if res_override is not None:
res = res_override.value
retbins = [XSDataDouble(x) for x in bins]
data_output = XSDataResCutoffResult()
data_output.res = XSDataDouble(res)
data_output.bins = retbins
totals = self.dataInput.total_completeness
data_output.total_complete = totals.complete
data_output.total_rfactor = totals.rfactor
data_output.total_isig = totals.isig
self.dataOutput = data_output
def __init__(self ):
"""
"""
EDPlugin.__init__(self )
self.setXSDataInputClass(XSDataResCutoff)
def process(self, _edObject = None):
EDPlugin.process(self)
output = XSDataXdsOutput()
# get all the file's contents, find where the info is and then
# use helper functions to retrieve stuff and put it in the
# data model
try:
f = open(self.dataInput.correct_lp.path.value, 'r')
lines = f.readlines()
except IOError:
EDVerbose.ERROR('Could not open the specified XDS output file for reading')
self.setFailure()
return
# look for the "big piece of information"
info_begin = None
info_end = None
for lineno, line in enumerate(lines):
if info_begin is None:
if line.find('REFINEMENT OF DIFFRACTION PARAMETERS USING ALL IMAGES') != -1:
info_begin = lineno
else:
if line.find('MEAN INTENSITY AS FUNCTION OF SPINDLE POSITION WITHIN DATA IMAGE') != -1:
info_end = lineno
break
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the refined parameters')
self.setFailure()
return
_extract_infos(lines[info_begin:info_end], output)
# second pass, look for the interesting table
info_begin = None
info_end = None
started = False
for line_no, line in enumerate(lines):
if line.find('REFLECTIONS OF TYPE H,0,0 0,K,0 0,0,L OR EXPECTED TO BE ABSENT (*)') != -1:
# the table will start shortly after
started = True
continue
if started:
# look if we are at the table yet
if line.find('LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected') != -1:
# there's an empty line after the header
info_begin = line_no + 2
if info_begin is not None and line.find('total') != -1:
# we're at the last table line
info_end = line_no
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the completeness table')
self.setFailure()
return
_extract_completeness_entries(lines[info_begin:info_end+1], output)
# now for the last bit: check if we were given a path to the
# gxparm file and if it exists get the space group and unit
# cell constants from it
if self.dataInput.gxparm is not None:
gxparm_path = self.dataInput.gxparm.path.value
if os.path.exists(gxparm_path):
with open(gxparm_path, 'r') as f:
lines = f.readlines()
for line in lines:
# the one we want has 7 floats
chunks = line.split()
if len(chunks) == 7:
output.sg_number = XSDataInteger(int(chunks[0]))
output.unit_cell_constants = [XSDataFloat(float(x)) for x in chunks[1:]]
input_file = self.dataInput.correct_lp.path.value
output.xds_run_directory = XSDataString(os.path.dirname(input_file))
self.dataOutput = output
def process(self, _edObject=None):
EDPlugin.process(self, _edObject)
EDVerbose.DEBUG("EDPluginWrapperForJobScheduler.process")
#print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
EDVerbose.DEBUG("Executing: oarsub \"python %s\"" % self.__strPathToPythonWrapper)
os.system("oarsub -l \"{cpu_vendor='INTEL'}/core=1\" \"python %s\"" % self.__strPathToPythonWrapper)
def __init__(self ):
"""
"""
EDPlugin.__init__(self )
self.setXSDataInputClass(XSDataXdsOutputFile)
def process(self, _edObject=None):
EDPlugin.process(self)
output = XSDataXdsOutput()
# get all the file's contents, find where the info is and then
# use helper functions to retrieve stuff and put it in the
# data model
try:
f = open(self.dataInput.correct_lp.path.value, 'r')
lines = f.readlines()
except IOError:
EDVerbose.ERROR(
'Could not open the specified XDS output file for reading')
self.setFailure()
return
# look for the "big piece of information"
info_begin = None
info_end = None
for lineno, line in enumerate(lines):
if info_begin is None:
if line.find(
'REFINEMENT OF DIFFRACTION PARAMETERS USING ALL IMAGES'
) != -1:
info_begin = lineno
else:
if line.find(
'MEAN INTENSITY AS FUNCTION OF SPINDLE POSITION WITHIN DATA IMAGE'
) != -1:
info_end = lineno
break
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the refined parameters')
self.setFailure()
return
_extract_infos(lines[info_begin:info_end], output)
# second pass, look for the interesting table
info_begin = None
info_end = None
started = False
for line_no, line in enumerate(lines):
if line.find(
'REFLECTIONS OF TYPE H,0,0 0,K,0 0,0,L OR EXPECTED TO BE ABSENT (*)'
) != -1:
# the table will start shortly after
started = True
continue
if started:
# look if we are at the table yet
if line.find(
'LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected'
) != -1:
# there's an empty line after the header
info_begin = line_no + 2
if info_begin is not None and line.find('total') != -1:
# we're at the last table line
info_end = line_no
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the completeness table')
self.setFailure()
return
_extract_completeness_entries(lines[info_begin:info_end + 1], output)
# now for the last bit: check if we were given a path to the
# gxparm file and if it exists get the space group and unit
# cell constants from it
if self.dataInput.gxparm is not None:
gxparm_path = self.dataInput.gxparm.path.value
if os.path.exists(gxparm_path):
with open(gxparm_path, 'r') as f:
lines = f.readlines()
for line in lines:
# the one we want has 7 floats
chunks = line.split()
if len(chunks) == 7:
output.sg_number = XSDataInteger(int(chunks[0]))
output.unit_cell_constants = [
XSDataFloat(float(x)) for x in chunks[1:]
]
input_file = self.dataInput.correct_lp.path.value
output.xds_run_directory = XSDataString(os.path.dirname(input_file))
self.dataOutput = output
