def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def preProcess(self):
"""
PreProcess of the execution test: download a set of images from http://www.edna-site.org
and remove any existing output file
"""
EDTestCasePluginExecute.preProcess(self)
self.loadTestImage([ "bioSaxsMask.edf", "bioSaxsNormalized.edf", "bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"])
strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(strExpectedOutput)
self.integratedCurve = xsDataResultReference.getIntegratedCurve().getPath().value
EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
if not os.path.isdir(os.path.dirname(self.integratedCurve)):
os.makedirs(os.path.dirname(self.integratedCurve))
if os.path.isfile(self.integratedCurve):
EDVerbose.DEBUG(" Output Integrated Curve file exists %s, I will remove it" % self.integratedCurve)
os.remove(self.integratedCurve)
EDUtilsParallel.initializeNbThread()
# self.correctedImage = xsDataResultReference.getCorrectedImage().getPath().value
# EDVerbose.DEBUG("Output Corrected Image file is %s" % self.correctedImage)
# if not os.path.isdir(os.path.dirname(self.correctedImage)):
# os.makedirs(os.path.dirname(self.correctedImage))
# if os.path.isfile(self.correctedImage):
# EDVerbose.DEBUG(" Output Corrected Image file exists %s, I will remove it" % self.correctedImage)
# os.remove(self.correctedImage)
if not pyFAI.version.startswith("0.7"):
EDVerbose.ERROR('pyFAI is not the right version, tested only with 0.7.2, here running version %s' % pyFAI.version)
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
if (_strXmlInput is None) or (_strXmlInput == "") :
return
#This is a trick to work-around bug #463:
# Run the fist thread alone (delay the second, third, ...)
# semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter
# I need the value of the semaphore to guess if the current call is the first or not.
# Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking.
# Them all other, limited by the semaphore.
if self.__bIsFirstExecute:
sys.stdout.write("Waiting for first thread to initialize ....")
while EDUtilsParallel.getNbRunning() > 0:
time.sleep(0.5)
sys.stdout.write(".")
EDUtilsParallel.semaphoreNbThreadsAcquire()
edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(self.__strPluginName)
if (edPlugin is not None):
edPlugin.setDataInput(_strXmlInput)
edPlugin.connectSUCCESS(self.successPluginExecution)
edPlugin.connectFAILURE(self.failurePluginExecution)
edPlugin.execute()
#edPlugin.executeSynchronous()
else:
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)
def preProcess(self):
"""
PreProcess of the execution test: download a set of images from http://www.edna-site.org
and remove any existing output file
"""
EDTestCasePluginExecute.preProcess(self)
self.loadTestImage([
"bioSaxsRaw.edf", "bioSaxsMask.edf", "bioSaxsCorrected.edf",
"bioSaxsProcessNormalized.edf", "bioSaxsProcessIntegrated.edf",
"bioSaxsProcessIntegrated1_2.dat"
])
strExpectedOutput = self.readAndParseFile(
self.getReferenceDataOutputFile())
EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(
strExpectedOutput)
self.refNormImg = xsDataResultReference.normalizedImage.path.value
self.refIntCrv = xsDataResultReference.integratedCurve.path.value
if not os.path.isdir(os.path.dirname(self.refNormImg)):
os.makedirs(os.path.dirname(self.refNormImg))
if os.path.isfile(self.refNormImg):
os.remove(self.refNormImg)
if os.path.isfile(self.refIntCrv):
os.remove(self.refIntCrv)
EDUtilsParallel.initializeNbThread()
def preProcess(self):
"""
PreProcess of the execution test: download a set of images from http://www.edna-site.org
and remove any existing output file
"""
EDTestCasePluginExecute.preProcess(self)
self.loadTestImage([
"bioSaxsMask.edf", "bioSaxsNormalized.edf",
"bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"
])
strExpectedOutput = self.readAndParseFile(
self.getReferenceDataOutputFile())
EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(
strExpectedOutput)
self.integratedCurve = xsDataResultReference.getIntegratedCurve(
).getPath().value
EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
if not os.path.isdir(os.path.dirname(self.integratedCurve)):
os.makedirs(os.path.dirname(self.integratedCurve))
if os.path.isfile(self.integratedCurve):
EDVerbose.DEBUG(
" Output Integrated Curve file exists %s, I will remove it" %
self.integratedCurve)
os.remove(self.integratedCurve)
EDUtilsParallel.initializeNbThread()
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
self.DEBUG("Acquire semaphore nbCPU by plugin %s" % self.getPluginName())
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
EDVerbose.DEBUG("Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
EDVerbose.DEBUG("Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
self.DEBUG("Release semaphore nbCPU by plugin %s" % self.getPluginName())
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
"""
Initialize the Plugin Unit test
"""
EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
self.__edPlugin = None
# To track dead locks
EDUtilsParallel.initializeNbThread(1)
def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
"""
Initialize the Plugin Unit test
"""
EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir,
_strTestName)
self.__edPlugin = None
# To track dead locks
EDUtilsParallel.initializeNbThread(1)
def preProcess(self, _edObject=None):
"""
preProcess of the plugin:
Ensure a CPU resource is available for the processing by acquiring a semaphore
"""
EDVerbose.DEBUG(
"Acquire semaphore nbCPU by plugin %s, currently value: %s" %
(self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsAcquire()
EDPlugin.preProcess(self, _edObject)
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
Release a CPU resource by releasing the semaphore
"""
EDVerbose.DEBUG(
"Release semaphore nbCPU by plugin %s, currently value: %s" %
(self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
EDUtilsParallel.semaphoreNbThreadsRelease()
EDPlugin.finallyProcess(self, _edObject)
def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def cleanUp(self, _edPlugin):
"""
Cleans up some empty directories
(mainly working directories that were created automatically, but not filled as Unit tests do not execute the plugin)
"""
workingDirectory = _edPlugin.getWorkingDirectory()
if (workingDirectory is not None):
fileList = glob.glob(os.path.join(workingDirectory, "*"))
if (len(fileList) == 0):
EDVerbose.DEBUG("Deleting " + workingDirectory + " ...")
shutil.rmtree(workingDirectory)
EDUtilsParallel.semaphoreNbThreadsRelease()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
if self.__functXMLerr is None:
EDVerbose.screen("Plugin %s execution ended with failure" % self.__strPluginName)
else:
self.__semaphoreErr.acquire()
self.__functXMLerr(_edObject.dataInput.marshal())
self.__semaphoreErr.release()
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
if self.__functXMLout is None:
EDVerbose.screen("Plugin %s execution ended with success" % self.__strPluginName)
else:
self.__semaphoreOut.acquire()
self.__functXMLout(_edObject.dataInput.marshal(), _edObject.getDataOutput().marshal())
self.__semaphoreOut.release()
def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU),
"Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def cleanUp(self, _edPlugin):
"""
Cleans up some empty directories
(mainly working directories that were created automatically, but not filled as Unit tests do not execute the plugin)
"""
workingDirectory = _edPlugin.getWorkingDirectory()
if(workingDirectory is not None):
fileList = glob.glob(os.path.join(workingDirectory, "*"))
if(len(fileList) == 0):
EDVerbose.DEBUG("Deleting " + workingDirectory + " ...")
shutil.rmtree(workingDirectory)
EDUtilsParallel.semaphoreNbThreadsRelease()
def executeControlledPlugins(self):
"""
Execute all plugins under control:
"""
bAllFinished = False
while not bAllFinished:
#acquire all semaphores to be sure no plugins are under configuration !
for sem in self.lstSem:
with sem:
pass
if self.queue.empty():
self.synchronizePlugins()
bAllFinished = self.queue.empty()
else:
while not self.queue.empty():
try:
plugin = self.queue.get()
except Exception:
self.WARNING("In EDPluginControlAlignStackv1_0, exception in self.queue.get()")
break
else:
#this is a hack to prevent thousands of threads to be launched at once.
with EDUtilsParallel.getSemaphoreNbThreads():
pass
plugin.execute()
self.synchronizePlugins()
def executeControlledPlugins(self):
"""
Execute all plugins under control:
"""
bAllFinished = False
while not bAllFinished:
#acquire all semaphores to be sure no plugins are under configuration !
for sem in self.lstSem:
with sem:
pass
if self.queue.empty():
self.synchronizePlugins()
bAllFinished = self.queue.empty()
else:
while not self.queue.empty():
try:
plugin = self.queue.get()
except Exception:
self.WARNING(
"In EDPluginControlAlignStackv1_0, exception in self.queue.get()"
)
break
else:
#this is a hack to prevent thousands of threads to be launched at once.
with EDUtilsParallel.getSemaphoreNbThreads():
pass
plugin.execute()
self.synchronizePlugins()
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
sys.stderr.write("Waiting for launched jobs to finish .")
while (EDUtilsParallel.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
sys.stderr.write("Waiting for launched jobs to finish .")
while (EDUtilsParallel.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
def __init__(self, strPluginName, iNbCpu=None):
EDLogging.__init__(self)
self.pluginName = strPluginName
self.startTime = time.time()
try:
self.iNbCpu = int(iNbCpu)
except:
self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()
self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
EDUtilsParallel.initializeNbThread(self.iNbCpu)
self.jobQueue = Queue()
self.processingSem = Semaphore()
self.statLock = Semaphore()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def join(self):
"""
wait for all jobs to finish
"""
while not (self.jobQueue.empty() and \
(self.__semaphoreNbThreads._Semaphore__value == self.iNbCpu) and \
(EDUtilsParallel.getNbRunning() == 0) and \
(self.processingSem._Semaphore__value == 1) and\
(len(EDStatus.getRunning()) == 0)):
time.sleep(1)
def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
"""
Initialise the Plugin Execution test
- Edna site
- Configuration files
- Data input file
- Plugin launcher
"""
EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
self._strRefConfigFile = None
self._dictStrDataInputFiles = {}
self._strDefaultInputDataKey = "defaultInputData"
self._dictStrReferenceDataOutputFiles = {}
self._strDefaultOutputDataKey = "defaultOutputData"
self._iNoExpectedErrorMessages = 0
self._iNoExpectedWarningMessages = 0
self._bAcceptPluginFailure = False
# Deprecated!
self.m_edObtainedOutputDataFile = None
EDUtilsParallel.uninitializeNbThread()
def preProcess(self):
"""
PreProcess of the execution test: download a set of images from http://www.edna-site.org
and remove any existing output file
"""
EDTestCasePluginExecute.preProcess(self)
self.loadTestImage(["bioSaxsRaw.edf", "bioSaxsMask.edf",
"bioSaxsCorrected.edf", "bioSaxsProcessNormalized.edf",
"bioSaxsProcessIntegrated.edf", "bioSaxsProcessIntegrated1_2.dat"])
strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(strExpectedOutput)
self.refNormImg = xsDataResultReference.normalizedImage.path.value
self.refIntCrv = xsDataResultReference.integratedCurve.path.value
if not os.path.isdir(os.path.dirname(self.refNormImg)):
os.makedirs(os.path.dirname(self.refNormImg))
if os.path.isfile(self.refNormImg):
os.remove(self.refNormImg)
if os.path.isfile(self.refIntCrv):
os.remove(self.refIntCrv)
EDUtilsParallel.initializeNbThread()
def chooseWorker(self):
"""
selects the worker and sets self.worker
"""
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
# self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
EDPluginSPDCorrectv10.__lockWorker.acquire()
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus()))
if (oneWorker.getConfig() == self._SPDconfig) and (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
if self.worker is None:
if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
self.worker = SPDworker()
EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
self.worker.setExecutable(self.getExecutable())
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID))
self.worker.initialize(self._SPDconfig)
self.worker.setTimeOut(self.getTimeOut())
while self.worker is None:
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
bConfigOK = False
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
#bug1 those string must specifically be the same ... change line 248 in spdCake1.5
if (oneWorker.getConfig() == self._SPDconfig):
bConfigOK = True
if (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
self.workerID = idx
if bConfigOK == False:
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
if (oneWorker.getStatus() in ["free", "uninitialized"]):
#Bug2 why doest this work ???
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
oneWorker.initialize(self._SPDconfig)
self.worker = oneWorker
self.workerID = idx
EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker))
time.sleep(0.1)
if self.workerID is None:
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
EDPluginSPDCorrectv10.__lockWorker.release()
def integrate(self):
if (self.integrator.getPyFAI() != self.integrator_config) or \
(self.integrator.wavelength != self.experimentSetup.wavelength.value):
self.screen("Resetting PyFAI integrator")
self.integrator.setPyFAI(**self.integrator_config)
self.integrator.wavelength = self.experimentSetup.wavelength.value
with EDUtilsParallel.getSemaphoreNbThreads():
img = fabio.open(self.normalizedImage)
variance = img.next()
q, I, std = self.integrator.saxs(data=img.data, nbPt=max(img.dim1, img.dim2),
correctSolidAngle=True,
variance=variance.data,
dummy= -2, delta_dummy=1.1,
method="splitBBox")
return q, I, std
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
self.jobQueue = Queue()
self.processingSem = threading.Semaphore()
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def integrate(self):
if (self.integrator.getPyFAI() != self.integrator_config) or \
(self.integrator.wavelength != self.experimentSetup.wavelength.value):
self.screen("Resetting PyFAI integrator")
self.integrator.setPyFAI(**self.integrator_config)
self.integrator.wavelength = self.experimentSetup.wavelength.value
with EDUtilsParallel.getSemaphoreNbThreads():
img = fabio.open(self.normalizedImage)
variance = img.next()
q, I, std = self.integrator.saxs(data=img.data, nbPt=max(img.dim1, img.dim2),
correctSolidAngle=True,
variance=variance.data,
dummy= -2, delta_dummy=1.1,
method="splitBBox")
return q, I, std
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." %
iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(
EDUtilsParallel.detectNumberOfCPUs())
self.jobQueue = Queue()
self.processingSem = threading.Semaphore()
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def process(self, _edPlugin=None):
"""
Executes the action cluster. This method will return only when
all actions have finished.
"""
self.DEBUG("EDActionCluster.process")
if (not self.__iNoThreads):
self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
for edAction in self.__lActions:
edAction.connectSUCCESS(self.__semaphoreRelease)
edAction.connectFAILURE(self.__setActionClusterFailure)
edAction.connectFAILURE(self.__semaphoreRelease)
self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName())
self.__semaphoreActionCluster.acquire()
edAction.execute()
# Wait for all threads to finish
for edAction in self.__lActions:
edAction.join()
def process(self, _edPlugin=None):
"""
Executes the action cluster. This method will return only when
all actions have finished.
"""
self.DEBUG("EDActionCluster.process")
if (not self.__iNoThreads):
self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
for edAction in self.__lActions:
edAction.connectSUCCESS(self.__semaphoreRelease)
edAction.connectFAILURE(self.__setActionClusterFailure)
edAction.connectFAILURE(self.__semaphoreRelease)
self.DEBUG("EDActionCluster.process : Starting action %s" %
edAction.getClassName())
self.__semaphoreActionCluster.acquire()
edAction.execute()
# Wait for all threads to finish
for edAction in self.__lActions:
edAction.join()
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
self.quit = False
self.jobQueue = Queue() # queue containing jobs to process
self.eventQueue = Queue() # queue containing finished jobs
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
self.processingThread = threading.Thread(target=self.startProcessing)
self.processingThread.start()
self.finishingThread = threading.Thread(target=self.process_event)
self.finishingThread.start()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlDozorv1_0.process")
self.sendMessageToMXCuBE("Processing started...", "info")
EDUtilsParallel.initializeNbThread()
xsDataResultControlDozor = XSDataResultControlDozor()
# Check if connection to ISPyB needed
if self.dataInput.dataCollectionId is not None:
edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
if self.batchSize is None:
batchSize = ispybDataCollection.numberOfImages
else:
batchSize = self.batchSize
if batchSize > self.maxBatchSize:
batchSize = self.maxBatchSize
if abs(ispybDataCollection.overlap) > 1:
self.hasOverlap = True
self.overlap = ispybDataCollection.overlap
dictImage = self.createImageDictFromISPyB(ispybDataCollection)
else:
# No connection to ISPyB, take parameters from input
if self.dataInput.batchSize is None:
batchSize = self.maxBatchSize
else:
batchSize = self.dataInput.batchSize.value
dictImage = self.createImageDict(self.dataInput)
self.screen("Dozor batch size: {0}".format(batchSize))
if self.dataInput.hdf5BatchSize is not None:
self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
# Convert HDF5 images to CBF
self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
if self.doRadiationDamage:
self.cbfTempDir = None
else:
self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
for listBatch in listAllBatches:
# Read the header from the first image in the batch
xsDataFile = dictImage[listBatch[0]]
edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = xsDataFile
edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
edPluginControlReadImageHeader.executeSynchronous()
subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
xsDataInputDozor = XSDataInputDozor()
beam = subWedge.experimentalCondition.beam
detector = subWedge.experimentalCondition.detector
goniostat = subWedge.experimentalCondition.goniostat
xsDataInputDozor.detectorType = detector.type
xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
xsDataInputDozor.spotSize = XSDataInteger(3)
xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
# xsDataInputDozor.fractionPolatization : XSDataDouble optional
orgx = detector.beamPositionY.value / detector.pixelSizeY.value
orgy = detector.beamPositionX.value / detector.pixelSizeX.value
xsDataInputDozor.orgx = XSDataDouble(orgx)
xsDataInputDozor.orgy = XSDataDouble(orgy)
xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
# xsDataInputDozor.imageStep : XSDataDouble optional
xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
xsDataInputDozor.firstImageNumber = subWedge.image[0].number
xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
if self.hasOverlap:
xsDataInputDozor.overlap = XSDataAngle(self.overlap)
strFileName = subWedge.image[0].path.value
strPrefix = EDUtilsImage.getPrefix(strFileName)
strSuffix = EDUtilsImage.getSuffix(strFileName)
if EDUtilsPath.isEMBL():
strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
elif self.hasHdf5Prefix and not self.hasOverlap:
strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
else:
strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
edPluginDozor.dataInput = xsDataInputDozor
edPluginDozor.execute()
edPluginDozor.synchronize()
indexImage = 0
imageDozorBatchList = []
for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
xsDataControlImageDozor = XSDataControlImageDozor()
xsDataControlImageDozor.number = xsDataResultDozor.number
xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
xsDataControlImageDozor.angle = xsDataResultDozor.angle
xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
if xsDataResultControlDozor.inputDozor is None:
xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
indexImage += 1
dozorSpotListShape = []
dozorSpotList = []
spotFile = None
if xsDataControlImageDozor.spotFile is not None:
spotFile = xsDataControlImageDozor.spotFile.path.value
if os.path.exists(spotFile):
numpyArray = numpy.loadtxt(spotFile, skiprows=3)
dozorSpotList = base64.b64encode(numpyArray.tostring())
dozorSpotListShape.append(numpyArray.shape[0])
if len(numpyArray.shape) > 1:
dozorSpotListShape.append(numpyArray.shape[1])
imageDozorDict = {"index": xsDataControlImageDozor.number.value,
"imageName": xsDataControlImageDozor.image.path.value,
"dozor_score": xsDataControlImageDozor.mainScore.value,
"dozorSpotsNumOf" : xsDataControlImageDozor.spotsNumOf.value,
"dozorSpotFile": spotFile,
"dozorSpotList" : dozorSpotList,
"dozorSpotListShape": dozorSpotListShape,
"dozorSpotsIntAver": xsDataControlImageDozor.spotsIntAver.value,
"dozorSpotsResolution": xsDataControlImageDozor.spotsResolution.value
}
imageDozorBatchList.append(imageDozorDict)
xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
xsDataResultControlDozor.pngPlots = edPluginDozor.dataOutput.pngPlots
self.sendResultToMXCuBE(imageDozorBatchList)
self.sendMessageToMXCuBE("Batch processed")
self.dataOutput = xsDataResultControlDozor
if self.cbfTempDir is not None:
if self.dataInput.keepCbfTmpDirectory is not None and self.dataInput.keepCbfTmpDirectory.value:
self.dataOutput.pathToCbfDirectory = XSDataFile(XSDataString(self.cbfTempDir))
else:
shutil.rmtree(self.cbfTempDir)
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process")
EDUtilsParallel.initializeNbThread()
# Check batch size
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
self.screen("Batch size: {0}".format(batchSize))
# Check if we should do distlSignalStrength:
bDoDistlSignalStrength = True
if self.dataInput.doDistlSignalStrength is not None:
if not self.dataInput.doDistlSignalStrength.value:
bDoDistlSignalStrength = False
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Loop through all the incoming reference images
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
listPluginDistl = []
listPluginDozor = []
listBatch = []
indexBatch = 0
listH5FilePath = []
ispybDataCollection = None
for xsDataImage in listXSDataImage:
strPathToImage = xsDataImage.path.value
# If Eiger, just wait for the h5 file
if "id30a3" in strPathToImage:
h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize)
# print(h5FilePath)
# print(hdf5ImageNumber)
if not h5DataFilePath in listH5FilePath:
self.screen("ID30a3 Eiger data, waiting for master and data files...")
listH5FilePath.append(h5DataFilePath)
self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
self.screen("Waiting for file {0}".format(h5DataFilePath))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
# hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
ispybDataCollection = None
time.sleep(1)
indexLoop = 1
continueLoop = True
while continueLoop:
xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage))
imageNumber = EDUtilsImage.getImageNumber(strPathToImage)
xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber)
xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber)
xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection
edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop))
edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
edPluginControlH5ToCBF.executeSynchronous()
cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
# print(cbfFile)
# print(indexLoop)
if cbfFile is not None:
strPathToImage = cbfFile.path.value
# print(cbfFile.path.value)
if os.path.exists(strPathToImage):
self.screen("Image has been converted to CBF file: {0}".format(strPathToImage))
continueLoop = False
# print(continueLoop)
if continueLoop:
self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage))
indexLoop += 1
time.sleep(5)
if indexLoop > 10:
continueLoop = False
ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection
elif not os.path.exists(strPathToImage):
self.screen("Waiting for file {0}".format(strPathToImage))
self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(strPathToImage):
strError = "Time-out while waiting for image %s" % strPathToImage
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
# Check if we should run distl.signalStrength
xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
edPluginPluginExecImageQualityIndicator = None
if bDoDistlSignalStrength:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew)
edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator))
listBatch.append(xsDataImageNew.copy())
if len(listBatch) == batchSize:
edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append((edPluginControlDozor, listBatch))
listBatch = []
if len(listBatch) > 0:
# Process the remaining images...
edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append([edPluginControlDozor, listBatch])
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()
for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
xsDataImageQualityIndicators.image = xsDataImage.copy()
if edPluginPluginExecImageQualityIndicator is not None:
edPluginPluginExecImageQualityIndicator.synchronize()
if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (edPluginControlDozor, listBatch) in listPluginDozor:
edPluginControlDozor.synchronize()
for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
if imageDozor.spotFile is not None:
if os.path.exists(imageDozor.spotFile.path.value):
numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3)
xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring()))
xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0]))
if len(numpyArray.shape) > 1:
xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1]))
xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
if edPluginControlDozor.dataOutput.inputDozor is not None:
self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
edPluginControlDozor.dataOutput.inputDozor.marshal())
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have dozor_score from all images:
has_dozor_score = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.dozor_score is None:
has_dozor_score = False
if has_dozor_score:
listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
else:
listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
xsDataSubWedge = xsDataResultReadImageHeader.subWedge
self.xsDataCollection = XSDataCollection()
self.xsDataCollection.addSubWedge(xsDataSubWedge)
xsDataIndexingInput = XSDataIndexingInput()
xsDataIndexingInput.setDataCollection(self.xsDataCollection)
xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput)
edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName)
self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator])
edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput)
edPluginMOSFLMIndexing.execute()
for tupleMOSFLM in self.listPluginMOSFLM:
edPluginMOSFLMIndexing = tupleMOSFLM[0]
xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
edPluginMOSFLMIndexing.synchronize()
if not edPluginMOSFLMIndexing.isFailure():
xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
EDUtilsParallel.initializeNbThread()
# Check batch size
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
# Check if we should do distlSignalStrength:
bDoDistlSignalStrength = True
if self.dataInput.doDistlSignalStrength is not None:
if not self.dataInput.doDistlSignalStrength.value:
bDoDistlSignalStrength = False
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Loop through all the incoming reference images
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
listPluginDistl = []
listPluginDozor = []
listBatch = []
indexBatch = 0
for xsDataImage in listXSDataImage:
self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
strError = "Time-out while waiting for image %s" % xsDataImage.path.value
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
# Check if we should run distl.signalStrength
edPluginPluginExecImageQualityIndicator = None
if bDoDistlSignalStrength:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator))
listBatch.append(xsDataImage.copy())
if len(listBatch) == batchSize:
edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append((edPluginControlDozor, listBatch))
listBatch = []
if len(listBatch) > 0:
# Process the remaining images...
edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append([edPluginControlDozor, listBatch])
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()
for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
xsDataImageQualityIndicators.image = xsDataImage.copy()
if edPluginPluginExecImageQualityIndicator is not None:
edPluginPluginExecImageQualityIndicator.synchronize()
if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (edPluginControlDozor, listBatch) in listPluginDozor:
edPluginControlDozor.synchronize()
for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
if edPluginControlDozor.dataOutput.inputDozor is not None:
self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
edPluginControlDozor.dataOutput.inputDozor.marshal())
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have dozor_score from all images:
has_dozor_score = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.dozor_score is None:
has_dozor_score = False
if has_dozor_score:
listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
else:
listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName)
xsDataInputLabelitIndexing = XSDataInputLabelitIndexing()
xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path))
edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing)
self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator])
edPluginLabelitIndexing.execute()
for tupleLabelit in self.listPluginLabelit:
edPluginLabelitIndexing = tupleLabelit[0]
xsDataResultControlImageQualityIndicator = tupleLabelit[1]
edPluginLabelitIndexing.synchronize()
if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None:
xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput()
xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
EDUtilsParallel.initializeNbThread()
# Check batch size
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
self.screen(
"Image quality indicators batch size: {0}".format(batchSize))
# Check if we should do distlSignalStrength:
bDoDistlSignalStrength = True
if self.dataInput.doDistlSignalStrength is not None:
if not self.dataInput.doDistlSignalStrength.value:
bDoDistlSignalStrength = False
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Check if fast mesh (for HDF5)
isFastMesh = False
if self.dataInput.fastMesh:
isFastMesh = self.dataInput.fastMesh.value
# Loop through all the incoming reference images
if len(self.dataInput.image) == 0:
directory = self.dataInput.directory.path.value
template = self.dataInput.template.value
startNo = self.dataInput.startNo.value
endNo = self.dataInput.endNo.value
listXSDataImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = os.path.join(directory, imageName)
xsDataImage = XSDataImage(path=XSDataString(imagePath),
number=XSDataInteger(index))
listXSDataImage.append(xsDataImage)
else:
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPluginDistl = []
listPluginDozor = []
listOfImagesInBatch = []
listOfAllBatches = []
indexBatch = 0
listH5FilePath = []
# Process data in batches
for xsDataImage in listXSDataImage:
listOfImagesInBatch.append(xsDataImage.copy())
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
# Loop over batches
for listOfImagesInBatch in listOfAllBatches:
# First wait for images
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# If Eiger, just wait for the h5 file
if strPathToImage.endswith(".h5"):
h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(
strPathToImage,
batchSize=batchSize,
isFastMesh=isFastMesh)
# print(h5FilePath)
# print(hdf5ImageNumber)
if not h5DataFilePath in listH5FilePath:
self.screen(
"ID30a3 Eiger data, waiting for master and data files..."
)
listH5FilePath.append(h5DataFilePath)
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(h5DataFilePath))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen(
"Waiting for file {0}".format(h5DataFilePath))
self.DEBUG("Wait file timeOut set to %f" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
# hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
time.sleep(1)
if not os.path.exists(h5DataFilePath):
strError = "Time-out while waiting for image %s" % h5DataFilePath
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if not os.path.exists(strPathToImage):
# self.screen("Waiting for file {0}".format(strPathToImage))
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(strPathToImage))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen("Wait file timeOut set to %.0f s" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(strPathToImage):
strError = "Time-out while waiting for image %s" % strPathToImage
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
if not self.isFailure():
strPathToFirstImage = listOfImagesInBatch[0].path.value
if strPathToImage.endswith(".h5"):
indexLoop = 1
continueLoop = True
while continueLoop:
directory = os.path.dirname(strPathToFirstImage)
firstImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[0].path.value)
lastImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
xsDataInputH5ToCBF = XSDataInputH5ToCBF()
xsDataInputH5ToCBF.hdf5File = XSDataFile(
listOfImagesInBatch[0].path)
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
xsDataInputH5ToCBF.startImageNumber = XSDataInteger(
firstImage)
xsDataInputH5ToCBF.endImageNumber = XSDataInteger(
lastImage)
xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(
XSDataString(directory))
edPluginH5ToCBF = self.loadPlugin(
"EDPluginH5ToCBFv1_1")
edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
edPluginH5ToCBF.execute()
edPluginH5ToCBF.synchronize()
outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
if outputCBFFileTemplate is not None:
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:06d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)))
strPathToImage = os.path.join(
directory, lastCbfFile)
# print(cbfFile.path.value)
if os.path.exists(strPathToImage):
# Rename all images
for image in listOfImagesInBatch:
image.path.value = image.path.value.replace(
".h5", ".cbf")
imageNumber = EDUtilsImage.getImageNumber(
image.path.value)
oldPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:06d}".format(imageNumber)))
newPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:04d}".format(imageNumber)))
os.rename(oldPath, newPath)
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:04d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
))
strPathToImage = os.path.join(
directory, lastCbfFile)
self.screen(
"Image has been converted to CBF file: {0}"
.format(strPathToImage))
continueLoop = False
# print(continueLoop)
if continueLoop:
self.screen(
"Still waiting for converting to CBF file: {0}"
.format(strPathToImage))
indexLoop += 1
time.sleep(5)
if indexLoop > 10:
continueLoop = False
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# Check if we should run distl.signalStrength
xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
xsDataImageNew.number = XSDataInteger(
EDUtilsImage.getImageNumber(image.path.value))
edPluginPluginExecImageQualityIndicator = None
if bDoDistlSignalStrength:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(
xsDataImageNew)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listPluginDistl.append(
(xsDataImageNew.copy(),
edPluginPluginExecImageQualityIndicator))
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor, "ControlDozor_{0}".format(
os.path.splitext(
os.path.basename(strPathToFirstImage))[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listOfImagesInBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
len(listOfImagesInBatch))
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append(
(edPluginControlDozor, list(listOfImagesInBatch)))
if not self.isFailure():
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for (xsDataImage,
edPluginPluginExecImageQualityIndicator) in listPluginDistl:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
xsDataImageQualityIndicators.image = xsDataImage.copy()
if edPluginPluginExecImageQualityIndicator is not None:
edPluginPluginExecImageQualityIndicator.synchronize()
if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
for (edPluginControlDozor, listBatch) in listPluginDozor:
edPluginControlDozor.synchronize()
# Check that we got at least one result
if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = os.path.basename(listBatch[0].path.value)
lastImage = os.path.basename(listBatch[-1].path.value)
self.screen(
"No dozor results! Re-executing Dozor for images {0} to {1}"
.format(firstImage, lastImage))
time.sleep(5)
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor,
"ControlDozor_reexecution_{0}".format(
os.path.splitext(firstImage)[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(
image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.executeSynchronous()
for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
if imageDozor.spotFile is not None:
if os.path.exists(
imageDozor.spotFile.path.value):
numpyArray = numpy.loadtxt(
imageDozor.spotFile.path.value,
skiprows=3)
xsDataImageQualityIndicators.dozorSpotList = XSDataString(
base64.b64encode(
numpyArray.tostring()))
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[0]))
if len(numpyArray.shape) > 1:
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[1]))
xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
if edPluginControlDozor.dataOutput.inputDozor is not None:
self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput(
).parseString(
edPluginControlDozor.dataOutput.
inputDozor.marshal())
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have dozor_score from all images:
has_dozor_score = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.dozor_score is None:
has_dozor_score = False
if has_dozor_score:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.dozor_score.value)
else:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[
-5:]:
if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
xsDataInputReadImageHeader = XSDataInputReadImageHeader(
)
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path
)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
edPluginLabelitIndexing = self.loadPlugin(
self.strIndexingLabelitPluginName)
xsDataInputLabelitIndexing = XSDataInputLabelitIndexing(
)
xsDataInputLabelitIndexing.image.append(
XSDataImage(
xsDataResultControlImageQualityIndicator.
image.path))
edPluginLabelitIndexing.setDataInput(
xsDataInputLabelitIndexing)
self.listPluginLabelit.append([
edPluginLabelitIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginLabelitIndexing.execute()
for tupleLabelit in self.listPluginLabelit:
edPluginLabelitIndexing = tupleLabelit[0]
xsDataResultControlImageQualityIndicator = tupleLabelit[1]
edPluginLabelitIndexing.synchronize()
if not edPluginLabelitIndexing.isFailure(
) and edPluginLabelitIndexing.dataOutput is not None:
xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput(
)
xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
xsDataResultLabelitIndexing)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
makeFile.write("LOADER = %s %s" %
(fortranCompiler, os.linesep))
# elif oneline.startswith("ARCHFLAGS"):
# makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep))
# elif oneline.startswith("ARCH"):
# makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep))
# elif oneline.startswith("RANLIB"):
# makeFile.write("RANLIB = ls %s" % (os.linesep))
elif oneline.startswith("BLASLIB"):
makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep))
else:
makeFile.write(oneline)
makeFile.close()
try:
installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except Exception:
EDVerbose.ERROR("Error for BLAS in the 'make' step")
if not os.path.isdir(
os.path.join(installBlas.getDestinationDirectory(), "lib")):
os.makedirs(
os.path.join(installBlas.getDestinationDirectory(), "lib"))
blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib")
shutil.copyfile(
os.path.join(installBlas.getSourceDirectory(), libblas),
os.path.join(blasPath, libblas))
################################################################################
# END of Blas / Start of Lapack
################################################################################
if lapackPath is None:
def semaphoreNbThreadsRelease(self):
"""Method to release the semaphore that controls the number of threads running concurrently"""
EDUtilsParallel.semaphoreNbThreadsRelease()
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process")
EDUtilsParallel.initializeNbThread()
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Loop through all the incoming reference images
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPlugin = []
for xsDataImage in listXSDataImage:
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
strError = "Time-out while waiting for image %s" % xsDataImage.path.value
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
edPluginControlBackground3D = self.loadPlugin(
self.strPluginNameControlBackground3D)
listPlugin.append([
edPluginPluginExecImageQualityIndicator,
edPluginControlBackground3D
])
xsDataInputControlBackground3D = XSDataInputControlBackground3D(
)
xsDataInputControlBackground3D.addImage(
XSDataFile(xsDataImage.path))
edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D
edPluginControlBackground3D.execute()
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for pluginPair in listPlugin:
edPluginPluginExecImageQualityIndicator = pluginPair[0]
edPluginControlBackground3D = pluginPair[1]
edPluginPluginExecImageQualityIndicator.synchronize()
edPluginControlBackground3D.synchronize()
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
if edPluginControlBackground3D.dataOutput.imageBackground != []:
xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[
0].estimate
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have background3D_estimate from all images:
has_background3D_estimate = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.background3D_estimate is None:
has_background3D_estimate = False
if has_background3D_estimate:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.
background3D_estimate.value)
else:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.
totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
xsDataSubWedge = xsDataResultReadImageHeader.subWedge
self.xsDataCollection = XSDataCollection()
self.xsDataCollection.addSubWedge(xsDataSubWedge)
xsDataIndexingInput = XSDataIndexingInput()
xsDataIndexingInput.setDataCollection(
self.xsDataCollection)
xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
xsDataIndexingInput)
edPluginMOSFLMIndexing = self.loadPlugin(
self.strIndexingMOSFLMPluginName)
self.listPluginMOSFLM.append([
edPluginMOSFLMIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginMOSFLMIndexing.setDataInput(
xsDataMOSFLMIndexingInput)
edPluginMOSFLMIndexing.execute()
for tupleMOSFLM in self.listPluginMOSFLM:
edPluginMOSFLMIndexing = tupleMOSFLM[0]
xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
edPluginMOSFLMIndexing.synchronize()
if not edPluginMOSFLMIndexing.isFailure():
xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(
xsDataMOSFLMOutput)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
if pthreadPath is None:
try:
installHDF5.configure("--prefix=%s" %
(installHDF5.getDestinationDirectory()))
except Exception:
EDVerbose.ERROR("Error in the configure step, no pthread")
else:
try:
installHDF5.configure(
"--prefix=%s --enable-threadsafe --with-pthread=%s" %
(installHDF5.getDestinationDirectory(), pthreadPath))
except Exception:
EDVerbose.ERROR("Error in the configure step, with pthread")
try:
installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except Exception:
EDVerbose.ERROR("Error in the 'make' step")
try:
installHDF5.make("install")
except Exception:
EDVerbose.ERROR("Error in the 'make install' step")
hdfPath = installHDF5.getDestinationDirectory()
installHDF5.cleanSources()
else:
hdfPath = os.path.dirname(hdfPath)
EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
install.checkPythonVersion()
install.dependency("numpy", "20090405-Numpy-1.3")
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process")
EDUtilsParallel.initializeNbThread()
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Loop through all the incoming reference images
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPlugin = []
for xsDataImage in listXSDataImage:
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
strError = "Time-out while waiting for image %s" % xsDataImage.path.value
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
listPlugin.append(edPluginPluginExecImageQualityIndicator)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for edPluginPluginExecImageQualityIndicator in listPlugin:
edPluginPluginExecImageQualityIndicator.synchronize()
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.bDoISPyBUpload:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=
lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value
)
for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
xsDataSubWedge = xsDataResultReadImageHeader.subWedge
self.xsDataCollection = XSDataCollection()
self.xsDataCollection.addSubWedge(xsDataSubWedge)
xsDataIndexingInput = XSDataIndexingInput()
xsDataIndexingInput.setDataCollection(
self.xsDataCollection)
xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
xsDataIndexingInput)
edPluginMOSFLMIndexing = self.loadPlugin(
self.strIndexingMOSFLMPluginName)
self.listPluginMOSFLM.append([
edPluginMOSFLMIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginMOSFLMIndexing.setDataInput(
xsDataMOSFLMIndexingInput)
edPluginMOSFLMIndexing.execute()
for tupleMOSFLM in self.listPluginMOSFLM:
edPluginMOSFLMIndexing = tupleMOSFLM[0]
xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
edPluginMOSFLMIndexing.synchronize()
if not edPluginMOSFLMIndexing.isFailure():
xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(
xsDataMOSFLMOutput)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self):
self.addTestMethod(self.unitTestMoveToTempDir)
self.addTestMethod(self.unitTestDetectNumberOfCPUs)
EDUtilsParallel.uninitializeNbThread()
installHDF5.downloadLibrary()
installHDF5.unZipArchive()
pthreadPath = installHDF5.searchCLib("libpthread.so")
EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
if pthreadPath is None:
try:
installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory()))
except:
EDVerbose.ERROR("Error in the configure step, no pthread")
else:
try:
installHDF5.configure("--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath))
except:
EDVerbose.ERROR("Error in the configure step, with pthread")
try:
installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except:
EDVerbose.ERROR("Error in the 'make' step")
try:
installHDF5.make("install")
except:
EDVerbose.ERROR("Error in the 'make install' step")
hdfPath = installHDF5.getDestinationDirectory()
installHDF5.cleanSources()
else:
hdfPath = os.path.dirname(hdfPath)
EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
install.checkPythonVersion()
install.dependency("numpy", "20090405-Numpy-1.3")
def process(self):
self.addTestMethod(self.unitTestMoveToTempDir)
self.addTestMethod(self.unitTestDetectNumberOfCPUs)
EDUtilsParallel.uninitializeNbThread()
strEdnaKernel = os.path.join(pyStrEdnaHomePath, "kernel", "src")
sys.path.append(strEdnaKernel)
if not "PYTHONPATH" in os.environ:
os.environ["PYTHONPATH"] = strEdnaKernel
else:
os.environ["PYTHONPATH"] += strEdnaKernel
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "src")
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "testsuite")
from EDJob import EDJob
from EDUtilsParallel import EDUtilsParallel
from EDVerbose import EDVerbose
from EDFactoryPluginStatic import EDFactoryPluginStatic
EDUtilsParallel.initializeNbThread()
EDFactoryPluginStatic.loadModule("XSDataDocumentation")
from XSDataDocumentation import XSDataString, XSDataFile, XSDataInputEpydoc, XSDataInteger
dictJobs = {}
def findPlugins(EDNAHome, EDPluginPrefix="EDPlugin", XSDataPrefix="XSData", pythonExtension=".py"):
"""
This function is the walker that goes through all directories in EDNA_HOME directory and searches for EDNA plugins ...
@param EDNAHome: the path of EDNA_HOME
@type EDNAHome: string
@param EDPluginPrefix: the start of the name of an EDNA Plugin
@type EDPluginPrefix: string
@param pythonExtension: the extension of an EDNA plugin, usually .py
