def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputAlignStack)
self.iFrames = []
self.npArrays = []
self.xsdMeasureOffset = None
self.hdf5ExtraAttributes = None
self.xsdHDF5File = None
self.xsdHDF5Internal = None
self.bAlwaysMOvsRef = True
self.bDoAlign = True
self.semAccumulator = Semaphore()
self.semMeasure = Semaphore()
self.semShift = Semaphore()
self.lstSem = [
self.locked(), self.semAccumulator, self.semMeasure, self.semShift
]
self.queue = Queue()
self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0"
self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0"
self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0"
self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1"
self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10"
def parse(self):
"""
parse options from command line
"""
parser = optparse.OptionParser()
parser.add_option("-V", "--version", dest="version", action="store_true",
help="print version of the program and quit", metavar="FILE", default=False)
parser.add_option("-v", "--verbose",
action="store_true", dest="debug", default=False,
help="switch to debug/verbose mode")
parser.add_option("-m", "--mask", dest="mask",
help="file containing the mask (for image reconstruction)", default=None)
parser.add_option("-M", "--mode", dest="mode",
help="Mode can be online/offline/all", default="offline")
parser.add_option("-o", "--out", dest="output",
help="file for log", default=None)
parser.add_option("-w", "--wavelength", dest="wavelength", type="float",
help="wavelength of the X-Ray beam in Angstrom", default=None)
parser.add_option("-e", "--energy", dest="energy", type="float",
help="energy of the X-Ray beam in keV (hc=%skeV.A)" % self.hc, default=None)
parser.add_option("-t", "--template", dest="template", type="str",
help="template XML file", default=None)
parser.add_option("-n", "--nbcpu", dest="nbcpu", type="int",
help="template XML file", default=self.nbcpu)
(options, args) = parser.parse_args()
# Analyse aruments and options
if options.version:
print("BioSaxs Azimuthal integration version %s" % __version__)
sys.exit(0)
if options.debug:
EDVerbose.setVerboseDebugOn()
self.debug = True
if options.output:
EDVerbose.setLogFileName(options.output)
if options.mask and os.path.isfile(options.mask):
self.maskfile = options.mask
if options.template and os.path.isfile(options.template):
self.xml = open(options.template).read()
if options.wavelength:
self.wavelength = 1e-10 * options.wavelength
elif options.energy:
self.wavelength = 1e-10 * self.hc / options.energy
if options.mode=="offline":
self.mode = "offline"
self.newerOnly = False
elif options.mode=="online":
self.mode = "dirwarch"
self.newerOnly = True
elif options.mode=="dirwatch":
self.mode = "dirwarch"
self.newerOnly = False
self.cpu_sem = Semaphore(options.nbcpu)
self.nbcpu = options.nbcpu
self.dataFiles = [f for f in args if os.path.isfile(f)]
if not self.dataFiles:
raise RuntimeError("Please provide datafiles or read the --help")
def __init__(self):
"""
Constructor of the main pure virtual class.
This constructor implements:
- the creation of the semaphore
- definition of timer object (uninitialized as potentially not used)
"""
object.__init__(self)
with self.__class__.__semaphoreId:
self.__class__.__iId_class += 1
self.__iId = self.__class__.__iId_class
self.__semaphore = Semaphore()
self.__fTimeInit = None
self.__fTimeEnd = None
self.__classname = None
def __init__(self):
object.__init__(self)
self.__pySemaphoreWrite = Semaphore()
self.__bIsVerbose = True
self.__bIsVerboseDebug = False
self.__bIsTest = False
self.__edLogFile = EDLogFile()
def __init__(self, runId, first_curve=None):
self.id = runId
self.buffer = None
self.first_curve = first_curve
self.frames = {} #key: id, value: HPLCframe instance
self.curves = []
self.for_buffer = []
self.hdf5_filename = None
self.hdf5 = None
self.chunk_size = 250
self.lock = Semaphore()
if first_curve:
self.files.append(first_curve)
self.max_size = None
self.time = None
self.gnom = None
self.Dmax = None
self.total = None
self.volume = None
self.Rg = None
self.Rg_Stdev = None
self.I0 = None
self.I0_Stdev = None
self.quality = None
self.q = None
self.size = None
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def __init__(self, strPluginName, iNbCpu=None):
EDLogging.__init__(self)
self.pluginName = strPluginName
self.startTime = time.time()
try:
self.iNbCpu = int(iNbCpu)
except:
self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()
self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
EDUtilsParallel.initializeNbThread(self.iNbCpu)
self.jobQueue = Queue()
self.processingSem = Semaphore()
self.statLock = Semaphore()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def __init__(self):
self.XML = "<XSDataInput>\
<normalizedImage><path><value>${FULLPATH}</value></path></normalizedImage>\
<correctedImage><path><value>${DIRDIRNAME}/2d/${BASENAME}.edf</value></path></correctedImage>\
<normalizedImageSize><value>4100000</value></normalizedImageSize>\
<integratedCurve><path><value>${DIRDIRNAME}/1d/${BASENAME}.edf</value></path></integratedCurve>\
<maskFile><path><value>${MASKFILE}</value></path></maskFile>\
<code><value>BSA</value></code>\
</XSDataInput>"
self.maskfile = None
self.dataFiles = []
self.wavelength = 1.0
self.debug = False
self.mode = "offline"
self.newerOnly = False
self.nbcpu = multiprocessing.cpu_count()
self.cpu_sem = Semaphore(self.nbcpu)
self.process_sem = Semaphore()
self.queue = Queue()
def __init__(self):
"""
Constructor of the main pure virtual class.
This constructor implements:
- the creation of the semaphore
- definition of timer object (uninitialized as potentially not used)
"""
object.__init__(self)
with self.__class__.__semaphoreId:
self.__class__.__iId_class += 1
self.__iId = self.__class__.__iId_class
self.__semaphore = Semaphore()
self.__fTimeInit = None
self.__fTimeEnd = None
self.__classname = None
def initializeNbThread(cls, _iNbThread=None):
"""
Class method:
Initialises a semaphore with the right number of threads
@param _iNbThread: the maximum number of threads concurrently running and CPU intensive
@type _iNbThread: integer
"""
with cls._semaphoreInit:
if cls._semaphoreNbThreads is None:
if _iNbThread is None:
_iNbThread = cls.detectNumberOfCPUs()
EDVerbose.DEBUG(
"Initializing EDUtilsParallel semaphoreNbThreads to %s" %
_iNbThread)
cls._iNbThreads = _iNbThread
cls._semaphoreNbThreads = Semaphore(_iNbThread)
else:
if cls._iNbThreads != _iNbThread:
EDVerbose.WARNING(
"cls._semaphoreNbThreads was already initialized to %s, not changing to %s"
% (cls._iNbThreads, _iNbThread))
class EDObject(object):
"""
Virtual base class for all EDNA Objects (classes).
It offers some synchronization and locking capabilities to make the code thread safe.
"""
__semaphoreId = Semaphore()
__iId_class = 0
def __init__(self):
"""
Constructor of the main pure virtual class.
This constructor implements:
- the creation of the semaphore
- definition of timer object (uninitialized as potentially not used)
"""
object.__init__(self)
with self.__class__.__semaphoreId:
self.__class__.__iId_class += 1
self.__iId = self.__class__.__iId_class
self.__semaphore = Semaphore()
self.__fTimeInit = None
self.__fTimeEnd = None
self.__classname = None
def getId(self):
return self.__iId
def getClassName(self):
"""
Retrieves the name of the class
@return: the name of the class
@rtype: string
"""
return self.__class__.__name__
def synchronizeOn(self):
"""
This method must be used in together with the method synchronizeOff().
This method makes the code threadsafe till the method synchronizeOff
is called.
"""
self.__semaphore.acquire()
def synchronizeOff(self):
"""
This method must be used in together with the method synchronizeOn().
"""
self.__semaphore.release()
def locked(self):
return self.__semaphore
def setTimeInit(self):
"""
Initializes the timer for the object
"""
if self.__fTimeInit is None:
self.__fTimeInit = time.time()
def getTimeInit(self):
"""
Retrieves the time of initialization
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeInit
def setTimeEnd(self):
"""
Set the end of calculation time for the given object
"""
if self.__fTimeEnd is None:
self.__fTimeEnd = time.time()
def getTimeEnd(self):
"""
Retrieves the time of end of task
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeEnd
def getRunTime(self):
"""
@returns: the RunTime for the given object
@rtype: float
"""
fRetrunRunTime = 0.0
if self.__fTimeInit is not None:
if self.__fTimeEnd is None:
fRetrunRunTime = time.time() - self.__fTimeInit
else:
fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
return fRetrunRunTime
class EDParallelExecute(EDLogging):
"""
A class helping to make a multi-threaded application from a plugin name and a list of files.
"""
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def runEDNA(self, _pyListInputPaths=[ "." ], _strMode="dirwatch", _bNewerOnly=False):
"""
This method runs the parallel execution on the list of directories.
@param _pyListInputPaths: the name of the directories to look after.
@type _pyListInputPaths: python list of strings
@param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented)
@type _strMode: python string
@param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them.
@type _bNewerOnly: boolean
"""
self.moveToTempDir()
self.__listInputPaths = _pyListInputPaths
if _strMode == "dirwatch":
self.watch_directories(_bNewerOnly)
elif _strMode == "inotify":
print("inotify online notify mode not yet implemented")
raise
else: #mode offline
self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True)
self.waitForAllProcessToFinish()
def moveToTempDir(self):
"""
Create a temporary directory and put all logs there
"""
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-')
self.screen("The log directory of EDNA will be in " + self.__strTempDir)
os.chdir(self.__strTempDir)
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
jobid = None
if _strXmlInput not in ["", None]:
job = EDJob(self.__strPluginName)
job.setDataInput(_strXmlInput)
job.connectFAILURE(self.failureJobExecution)
job.connectSUCCESS(self.successJobExecution)
job.connectCallBack(self.unregisterJob)
self.semaphoreNbThreadsAcquire()
jobid = job.execute()
self.DEBUG("Running Job id %s" % jobid)
if jobid is None:
self.semaphoreNbThreadsRelease()
return jobid
def successJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with success
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.successJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLout is not None:
job = EDJob.getJobFromID(_jobId)
self.__functXMLout(job.getPlugin().getDataInput(), job.getPlugin().getDataOutput())
def failureJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with failure
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLerr is not None:
self.__functXMLerr(EDJob.getJobFromID(_jobId).getPlugin().getDataInput())
def unregisterJob(self, _jobid):
"""
remove the filename from the list of files currently under processing
@param _jobId: string of type EDPluginName-number
"""
with self.locked():
for oneKey in self.__dictCurrentlyRunning.copy():
if self.__dictCurrentlyRunning[oneKey] == _jobid:
self.__dictCurrentlyRunning.pop(oneKey)
def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False):
"""
This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode.
@param _listNewFiles: list of files newly created in the directory.
@type _listNewFiles: python list of strings.
@param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online.
@type _bIncludeSubdirs: boolean
"""
for oneFile in _listNewFiles:
if os.path.isdir(oneFile) and _bIncludeSubdirs == True:
for root, _, onesubdirfiles in os.walk(oneFile):
for onesubdirfile in onesubdirfiles:
strFilename = os.path.abspath(os.path.join(root, onesubdirfile))
if self.__bQuit == True:
return
self.processOneFile(strFilename)
elif os.path.isfile(oneFile):
if self.__bQuit == True:
return
self.processOneFile(oneFile)
def processOneFile(self, _strFilename):
"""
Process on file by calling subsequently the XML generator and the start method unless this file
is already under process (can happend with the watch_directory method).
@param _strFilename: filename to process
@type _strFilename: string
"""
if _strFilename not in self.__dictCurrentlyRunning:
with self.locked():
self.__dictCurrentlyRunning[_strFilename] = self.__strPluginName
strXmlData = self.__functXMLin(_strFilename)
if strXmlData in [None, ""]:
self.log("Not processing % s" % _strFilename)
with self.locked():
self.__dictCurrentlyRunning.pop(_strFilename)
else:
self.screen("Processing % s" % _strFilename)
jobid = self.start(strXmlData)
with self.locked():
if jobid is None:
self.__dictCurrentlyRunning.pop(_strFilename)
else:
self.__dictCurrentlyRunning[_strFilename] = jobid
def watch_directories (self, _bNewerOnly=False):
"""
Continuously monitors the paths and their subdirectories
for changes. If any files or directories are modified,
the callable function ( here the method self.runEdnaFunction() ) is called
with a list of the modified paths of both
files and directories. This function can return a Boolean value
for rescanning; if it returns True, the directory tree will be
rescanned without calling the function for any found changes.
(This is so this function can write changes into the tree and prevent itself
from being immediately called again.)
@param _bNewerOnly : Do you want to process only newer files
@type _bNewerOnly : Boolean
Basic principle: pyDictAllFiles is a dictionary mapping paths to
modification times. We repeatedly crawl through the directory
tree rooted at 'path', doing a stat() on each file and comparing
the modification time.
"""
dictAllFiles = {}
dictRemainingFiles = {}
listChangedFiles = []
def internalUpdateDict (unused, dirname, files):
"Traversal function for directories"
for strFilename in files:
path = os.path.join(dirname, strFilename)
try:
tempStat = os.stat(path)
except os.error:
# If a file has been deleted between os.path.walk()
# scanning the directory and now, we'll get an
# os.error here. Just ignore it -- we'll report
# the deletion on the next pass through the main loop.
continue
mtime = dictRemainingFiles.get(path)
if mtime is not None:
# Record this file as having been seen
del dictRemainingFiles[path]
# File's mtime has been changed since we last looked at it.
if tempStat.st_mtime > mtime:
listChangedFiles.append(path)
else:
# No recorded modification time, so it must be
# a brand new file.
listChangedFiles.append(path)
# Record current mtime of file.
dictAllFiles[path] = tempStat.st_mtime
if _bNewerOnly:
for path in self.__listInputPaths:
os.path.walk(path, internalUpdateDict, None)
# Main loop
rescan = False
while not self.__bQuit:
listChangedFiles = []
dictRemainingFiles = dictAllFiles.copy()
dictAllFiles = {}
for path in self.__listInputPaths:
os.path.walk(path, internalUpdateDict, None)
#removed_list = dictRemainingFiles.keys()
if rescan:
rescan = False
elif listChangedFiles:
rescan = self.runEdnaFunction(listChangedFiles, _bIncludeSubdirs=False)
time.sleep(self.__fDelay)
print("Quitting the online mode.")
def handleKill(self, signum, frame):
"""
This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program
"""
self.__bQuit = True
sys.stderr.write("Exit requested by signal %s with frame %s.\n" % (signum, frame))
self.waitForAllProcessToFinish()
os.chdir(self.__strCurrWorkDir)
def flush(self):
"""
This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work
"""
bFinished = False
while not bFinished:
xml = None
try:
xml = self.__functXMLin(None, flush=True)
except TypeError:
try:
xml = self.__functXMLin("", flush=True)
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
xml = None
if (xml is None) or (xml == ""):
bFinished = True
else:
self.screen ("Flushing data ...")
self.start(xml)
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
self.screen("Waiting for launched jobs to finish .")
while (self.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
EDJob.stats()
def cleanUp(self, listMethods=[]):
"""
Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD)
@param listMethods: allows to finish some things in the plugin.
@type listMethods: list of strings representing names of methods of the plugin to be called.
"""
self.waitForAllProcessToFinish()
for strOneMethod in listMethods:
try:
print("calling edPlugin.%s" % strOneMethod)
exec("edPlugin.%s" % strOneMethod)
except Exception:
print("error in processing %s" % strOneMethod)
################################################################################
# Nota: there are 2 levels of controls for the number of thread currently running:
# * One here to limit the number of control plugin running at once
# * One on the Exec plugin level for finer grain optimisation
################################################################################
def semaphoreNbThreadsAcquire(self):
"""Method to acquire the semaphore that controls the number of plugins running concurrently"""
# pass
self.__semaphoreNbThreads.acquire()
def semaphoreNbThreadsRelease(self):
"""Method to release the semaphore that controls the number of plugins running concurrently"""
# pass
self.__semaphoreNbThreads.release()
def getNbRunning(self):
"""
Class method:
getter for the number of CPU-active threads running
@return: the number of CPU-active threads runnings
@rtype: integer
"""
#return EDUtilsParallel.getNbRunning()
return self.__iNbThreads - self.__semaphoreNbThreads._Semaphore__value
class Reprocess(EDLogging):
def __init__(self, strPluginName, iNbCpu=None):
EDLogging.__init__(self)
self.pluginName = strPluginName
self.startTime = time.time()
try:
self.iNbCpu = int(iNbCpu)
except:
self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()
self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
EDUtilsParallel.initializeNbThread(self.iNbCpu)
self.jobQueue = Queue()
self.processingSem = Semaphore()
self.statLock = Semaphore()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def startJob(self, xsd):
"""
@param xsd: XML data structure as a string or path to a string
@return: jobID which is a sting: Plugin-000001
"""
self.DEBUG("In %s.startJob()" % self.__class__.__name__)
if type(xsd) in types.StringTypes:
if xsd.strip() == "":
return
if os.path.isfile(xsd):
xsd = open(xsd, "rb").read()
edJob = EDJob(self.pluginName)
if edJob is None:
return "Error in load Plugin"
jobId = edJob.getJobId()
edJob.setDataInput(xsd)
self.jobQueue.put(edJob)
if self.processingSem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
return jobId
def startProcessing(self):
"""
Process all jobs in the queue.
"""
with self.processingSem:
while not self.jobQueue.empty():
self.__semaphoreNbThreads.acquire()
edJob = self.jobQueue.get()
edJob.connectSUCCESS(self.successJobExecution)
edJob.connectFAILURE(self.failureJobExecution)
edJob.execute()
# edJob.synchronize()
def successJobExecution(self, jobId):
self.DEBUG("In %s.successJobExecution(%s)" % (self.__class__.__name__, jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastSuccess = jobId
gc.collect()
def failureJobExecution(self, jobId):
self.DEBUG("In %s.failureJobExecution(%s)" % (self.__class__.__name__, jobId))
with self.locked():
self.__semaphoreNbThreads.release()
EDJob.cleanJobfromID(jobId, False)
self.lastFailure = jobId
sys.stdout.flush()
sys.stderr.flush()
gc.collect()
def getRunning(self):
"""
retrieve the list of plugins currently under execution (with their plugin-Id)
"""
return EDStatus.getRunning()
def getSuccess(self):
"""
retrieve the list of plugins finished with success (with their plugin-Id)
"""
return EDStatus.getSuccess()
def getFailure(self):
"""
retrieve the list of plugins finished with failure (with their plugin-Id)
"""
return EDStatus.getFailure()
def collectStatistics(self):
"""
Retrieve some statistics on all EDNA-Jobs
@return: a page of information about EDNA-jobs
"""
t = threading.Thread(target=self.statistics)
t.start()
def statistics(self):
"""
retrieve some statistics about past jobs.
"""
with self.statLock:
fStartStat = time.time()
self.lastStatistics = EDJob.stats()
self.lastStatistics += os.linesep + "Statistics collected on %s, the collect took: %.3fs" % (time.asctime(), time.time() - fStartStat)
return self.lastStatistics
def getStatistics(self):
"""
just return statistics previously calculated
"""
return self.lastStatistics
def getJobOutput(self, jobId):
"""
Retrieve XML output form a job
@param jobId: name of the job
@return: output from a job
"""
return EDJob.getDataOutputFromId(jobId)
def getJobInput(self, jobId):
"""
Retrieve XML input from a job
@param jobId: name of the job
@return: xml input from a job
"""
return EDJob.getDataInputFromId(jobId)
def join(self):
"""
wait for all jobs to finish
"""
while not (self.jobQueue.empty() and \
(self.__semaphoreNbThreads._Semaphore__value == self.iNbCpu) and \
(EDUtilsParallel.getNbRunning() == 0) and \
(self.processingSem._Semaphore__value == 1) and\
(len(EDStatus.getRunning()) == 0)):
time.sleep(1)
class EDPluginBioSaxsProcessOneFilev1_4(EDPluginControl):
"""
Control plugin that does the same as previously without sub-plugin call ...
except WaitFile which is still called.
Nota normalization is done AFTER integration not before as previously
New in version 1.4: return I/Q/stderr via XSDataArrays
"""
cpWaitFile = "EDPluginWaitFile"
integrator = pyFAI.AzimuthalIntegrator()
integrator.wavelength = 1e-10
CONF_DUMMY_PIXEL_VALUE = "DummyPixelValue"
CONF_DUMMY_PIXEL_DELTA = "DummyPixelDelta"
CONF_OPENCL_DEVICE = "DeviceType"
__configured = False
dummy = -2
delta_dummy = 1.1
semaphore = Semaphore()
maskfile = None
if pyFAI.opencl.ocl is None:
METHOD = "lut"
else:
METHOD = "lut_ocl_gpu"
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputBioSaxsProcessOneFilev1_0)
self.__edPluginWaitFile = None
self.rawImage = None
self.rawImageSize = XSDataInteger(1024)
self.normalizedImage = None
self.integratedCurve = None
self.integratedImage = None
self.lstExecutiveSummary = []
self.sample = None
self.experimentSetup = None
self.integrator_config = {}
self.scale = None
self.detector = None
self.xsDataResult = XSDataResultBioSaxsProcessOneFilev1_0()
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image provided")
self.checkMandatoryParameters(self.dataInput.sample, "No sample information provided")
self.checkMandatoryParameters(self.dataInput.experimentSetup, "No experimental setup provided")
self.checkMandatoryParameters(self.dataInput.integratedCurve, "Missing integratedCurve")
def configure(self):
"""
Configures the plugin from the configuration file with the following parameters:
- DummyPixelValue: the value be assigned to dummy pixels.
- DummyPixelDelta: the value be assigned to delta dummy.
- DeviceType: "lut_ocl_1,3" will select device #3 on first platform #1
"""
EDPluginControl.configure(self)
if not self.__configured:
with self.semaphore:
if not self.__configured:
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.configure")
dummy = self.config.get(self.CONF_DUMMY_PIXEL_VALUE)
if dummy is None:
strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
%s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_VALUE, self.dummy)
self.WARNING(strMessage)
self.addErrorWarningMessagesToExecutiveSummary(strMessage)
else:
self.__class__.dummy = float(dummy)
ddummy = self.config.get(self.CONF_DUMMY_PIXEL_DELTA)
if ddummy is None:
strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
%s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_DELTA, self.delta_dummy)
self.WARNING(strMessage)
self.addErrorWarningMessagesToExecutiveSummary(strMessage)
else:
self.__class__.delta_dummy = float(ddummy)
method = self.config.get(self.CONF_OPENCL_DEVICE)
if method is None:
strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
%s, defaulting to "%s"' % (self.getBaseName(), self.CONF_OPENCL_DEVICE, self.METHOD)
self.WARNING(strMessage)
self.addErrorWarningMessagesToExecutiveSummary(strMessage)
else:
self.__class__.METHOD = method
self.__class__.__configured = True
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.preProcess")
self.__edPluginWaitFile = self.loadPlugin(self.cpWaitFile)
if self.dataInput.rawImageSize is not None:
self.rawImageSize = self.dataInput.rawImageSize
if self.dataInput.rawImage is not None:
self.rawImage = self.dataInput.rawImage.path.value
self.integratedCurve = self.dataInput.integratedCurve.path.value
curveDir = os.path.dirname(self.integratedCurve)
if not os.path.isdir(curveDir):
try:
os.mkdir(curveDir)
except OSError:
# could occur in race condition ...
pass
self.sample = self.dataInput.sample
self.experimentSetup = self.dataInput.experimentSetup
self.detector = self.experimentSetup.detector.value
if self.detector.lower() == "pilatus":
self.detector = "Pilatus1M"
else:
self.detector = self.detector.capitalize()
self.integrator_config = {'dist': self.experimentSetup.detectorDistance.value,
'pixel1': self.experimentSetup.pixelSize_2.value, # flip X,Y
'pixel2': self.experimentSetup.pixelSize_1.value, # flip X,Y
'poni1': self.experimentSetup.beamCenter_2.value * self.experimentSetup.pixelSize_2.value,
'poni2': self.experimentSetup.beamCenter_1.value * self.experimentSetup.pixelSize_1.value,
'rot1': 0.0,
'rot2': 0.0,
'rot3': 0.0,
'splineFile': None,
'detector': self.detector
}
i0 = self.experimentSetup.beamStopDiode.value
if i0 == 0:
warn = "beamStopDiode is Null --> If we are testing, this is OK, else investigate !!!"
self.lstExecutiveSummary.append(warn)
self.warning(warn)
self.scale = self.experimentSetup.normalizationFactor.value
else:
self.scale = self.experimentSetup.normalizationFactor.value / i0
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.process")
xsd = XSDataInputWaitFile(expectedFile=XSDataFile(XSDataString(self.rawImage)),
expectedSize=self.rawImageSize,
timeOut=XSDataTime(30))
self.__edPluginWaitFile.setDataInput(xsd)
self.__edPluginWaitFile.connectSUCCESS(self.doSuccessWaitFile)
self.__edPluginWaitFile.connectFAILURE(self.doFailureWaitFile)
self.__edPluginWaitFile.executeSynchronous()
if self.isFailure():
return
self.xsDataResult.sample = self.sample
self.xsDataResult.experimentSetup = self.experimentSetup
q, I, std = self.integrate()
I = self.normalize(I)
std = self.normalize(std)
self.write3ColumnAscii(q, I, std, self.integratedCurve)
self.xsDataResult.dataQ = EDUtilsArray.arrayToXSData(q)
self.xsDataResult.dataI = EDUtilsArray.arrayToXSData(I)
self.xsDataResult.dataStdErr = EDUtilsArray.arrayToXSData(std)
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.postProcess")
# Create some output data
if os.path.exists(self.integratedCurve):
self.xsDataResult.integratedCurve = XSDataFile(XSDataString(self.integratedCurve))
self.xsDataResult.status = XSDataStatus(executiveSummary=XSDataString(os.linesep.join(self.lstExecutiveSummary)))
self.setDataOutput(self.xsDataResult)
def integrate(self):
img = fabio.open(self.rawImage)
if "Date" in img.header:
self.experimentSetup.timeOfFrame = XSDataTime(time.mktime(time.strptime(img.header["Date"], "%a %b %d %H:%M:%S %Y")))
wavelength = EDUtilsUnit.getSIValue(self.experimentSetup.wavelength)
current_config = self.integrator.getPyFAI()
short_config = {}
for key in self.integrator_config:
short_config[key] = current_config[key]
with self.__class__.semaphore:
if (short_config != self.integrator_config) or \
(self.integrator.wavelength != wavelength) or\
(self.maskfile != self.experimentSetup.maskFile.path.value):
self.screen("Resetting PyFAI integrator")
self.integrator.setPyFAI(**self.integrator_config)
self.integrator.wavelength = wavelength
self.integrator.detector.mask = self.calc_mask()
q, I, std = self.integrator.integrate1d(img.data, max(img.dim1, img.dim2),
correctSolidAngle=True,
dummy=self.dummy, delta_dummy=self.delta_dummy,
filename=None,
error_model="poisson",
radial_range=None, azimuth_range=None,
polarization_factor=0, dark=None, flat=None,
method=self.METHOD, unit="q_nm^-1", safe=False)
self.lstExecutiveSummary.append("Azimuthal integration of raw image '%s'-->'%s'." % (self.rawImage, self.integratedCurve))
return q, I, std
def normalize(self, data):
"""
Perform the normalization of some data
@return: normalized data
"""
maskedData = numpy.ma.masked_array(data, abs(data - self.dummy) < self.delta_dummy)
return numpy.ma.filled(maskedData * self.scale, self.dummy)
def calc_mask(self):
"""
Merge the natural mask from the detector with the user proided one.
@return: numpy array with the mask
"""
mask = fabio.open(self.experimentSetup.maskFile.path.value).data
detector_mask = pyFAI.detectors.detector_factory(self.detector).calc_mask()
shape0, shape1 = detector_mask.shape
if detector_mask.shape == mask.shape:
mask = numpy.logical_or(mask, detector_mask)
else:
# crop the user defined mask
mask = numpy.logical_or(mask[:shape0, :shape1], detector_mask)
self.__class__.maskfile = self.experimentSetup.maskFile.path.value
return mask
def write3ColumnAscii(self, npaQ, npaI, npaStd=None, outputCurve="output.dat", hdr="#", linesep=os.linesep):
"""
@param npaQ,npaI,npaStd: 3x 1d numpy array containing Scattering vector, Intensity and deviation
@param outputCurve: name of the 3-column ascii file to be written
@param hdr: header mark, usually '#'
Adam Round explicitelly asked for (email from Date: Tue, 04 Oct 2011 15:22:29 +0200) :
Modification from:
# BSA buffer
# Sample c= 0.0 mg/ml (these two lines are required for current DOS pipeline and can be cleaned up once we use EDNA to get to ab-initio models)
#
# Sample environment:
# Detector = Pilatus 1M
# PixelSize_1 = 0.000172
# PixelSize_2 = 6.283185 (I think it could avoid confusion if we give teh actual pixel size as 0.000172 for X and Y and not to give the integrated sizes. Also could there also be a modification for PixelSize_1 as on the diagonal wont it be the hypotenuse (0.000243)? and thus will be on average a bit bigger than 0.000172)
#
# title = BSA buffer
# Frame 7 of 10
# Time per frame (s) = 10
# SampleDistance = 2.43
# WaveLength = 9.31e-11
# Normalization = 0.0004885
# History-1 = saxs_angle +pass -omod n -rsys normal -da 360_deg -odim = 1 /data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/2d/dumdum_008_07.edf/data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/misc/dumdum_008_07.ang
# DiodeCurr = 0.0001592934
# MachCurr = 163.3938
# Mask = /data/id14eh3/archive/CALIBRATION/MASK/Pcon_01Jun_msk.edf
# SaxsDataVersion = 2.40
#
# N 3
# L q*nm I_BSA buffer stddev
#
# Sample Information:
# Storage Temperature (degrees C): 4
# Measurement Temperature (degrees C): 10
# Concentration: 0.0
# Code: BSA
s-vector Intensity Error
s-vector Intensity Error
s-vector Intensity Error
s-vector Intensity Error
"""
hdr = str(hdr)
headers = []
if self.sample.comments is not None:
headers.append(hdr + " " + self.sample.comments.value)
else:
headers.append(hdr)
if self.sample.concentration is not None:
headers.append(hdr + " Sample c= %s mg/ml" % self.sample.concentration.value)
else:
headers.append(hdr + " Sample c= -1 mg/ml")
headers += [hdr,
hdr + " Sample environment:"]
if self.experimentSetup.detector is not None:
headers.append(hdr + " Detector = %s" % self.experimentSetup.detector.value)
if self.experimentSetup.pixelSize_1 is not None:
headers.append(hdr + " PixelSize_1 = %s" % self.experimentSetup.pixelSize_1.value)
if self.experimentSetup.pixelSize_2 is not None:
headers.append(hdr + " PixelSize_2 = %s" % self.experimentSetup.pixelSize_2.value)
headers.append(hdr)
if self.sample.comments is not None:
headers.append(hdr + " title = %s" % self.sample.comments.value)
if (self.experimentSetup.frameNumber is not None) and\
(self.experimentSetup.frameMax is not None):
headers.append(hdr + " Frame %s of %s" % (self.experimentSetup.frameNumber.value, self.experimentSetup.frameMax.value))
if self.experimentSetup.exposureTime is not None:
headers.append(hdr + " Time per frame (s) = %s" % self.experimentSetup.exposureTime.value)
if self.experimentSetup.detectorDistance is not None:
headers.append(hdr + " SampleDistance = %s" % self.experimentSetup.detectorDistance.value)
if self.experimentSetup.wavelength is not None:
headers.append(hdr + " WaveLength = %s" % self.experimentSetup.wavelength.value)
if self.experimentSetup.normalizationFactor is not None:
headers.append(hdr + " Normalization = %s" % self.experimentSetup.normalizationFactor.value)
if self.experimentSetup.beamStopDiode is not None:
headers.append(hdr + " DiodeCurr = %s" % self.experimentSetup.beamStopDiode.value)
if self.experimentSetup.machineCurrent is not None:
headers.append(hdr + " MachCurr = %s" % self.experimentSetup.machineCurrent.value)
if self.experimentSetup.maskFile is not None:
headers.append(hdr + " Mask = %s" % self.experimentSetup.maskFile.path.value)
headers.append(hdr)
headers.append(hdr + " N 3")
if self.sample.comments is not None:
headers.append(hdr + " L q*nm I_%s stddev" % self.sample.comments.value)
else:
headers.append(hdr + " L q*nm I_ stddev")
headers.append(hdr)
headers.append(hdr + " Sample Information:")
if self.experimentSetup.storageTemperature is not None:
headers.append(hdr + " Storage Temperature (degrees C): %s" % self.experimentSetup.storageTemperature.value)
if self.experimentSetup.exposureTemperature is not None:
headers.append(hdr + " Measurement Temperature (degrees C): %s" % self.experimentSetup.exposureTemperature.value)
if self.sample.concentration is not None:
headers.append(hdr + " Concentration: %s" % self.sample.concentration.value)
else:
headers.append(hdr + " Concentration: -1")
if self.sample.code is not None:
headers.append(hdr + " Code: %s" % self.sample.code.value)
else:
headers.append(hdr + " Code: ")
with open(outputCurve, "w") as f:
f.writelines(linesep.join(headers))
f.write(linesep)
if npaStd is None:
data = ["%14.6e %14.6e " % (q, I)
for q, I in zip(npaQ, npaI)
if abs(I - self.dummy) > self.delta_dummy]
else:
data = ["%14.6e %14.6e %14.6e" % (q, I, std)
for q, I, std in zip(npaQ, npaI, npaStd)
if abs(I - self.dummy) > self.delta_dummy]
data.append("")
f.writelines(linesep.join(data))
f.flush()
def doSuccessWaitFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile")
self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile")
def doFailureWaitFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile")
self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile")
self.lstExecutiveSummary.append("Timeout in waiting for file '%s'" % (self.rawImage))
self.setFailure()
class EDUtilsPath:
"""
This is a static utility class for handling of paths.
"""
EDNA_HOME = dirname(dirname(dirname(abspath(__file__))))
os.environ["EDNA_HOME"] = EDNA_HOME
_EDNA_SITE = None
_EDNA_USERTEMPFOLDER = None
_EDNA_TESTIMAGES = None
_EDNA_TESTS = None
_EDNA_TESTDATA = None
_USER = os.getenv("USER", "UndefindedUser")
_TMPDIR = os.getenv("TMPDIR", tempfile.gettempdir())
_EDNA_PROJECTS = {} # key: project name. Value: path to the project root
_EDNA_PLUGINCACHE = None
__semaphore = Semaphore()
@classmethod
def appendListOfPaths(cls, _strPath, _listOfPaths):
"""
Appends a list of paths to a path, for example [ "..", "..", "..", "..", "src" ],
and returns the absolute path.
"""
return abspath(os.path.join(_strPath, *_listOfPaths))
@classmethod
def mergePath(cls, _strPath1, _strPath2):
"""
Merges two paths and returns the absolute path.
Deprecated: please use 'os.path.join' instead
"""
EDVerbose.log(
"EDUtilsPath.mergePath is deprecated, please use 'os.path.join' instead"
)
strNewPath = os.path.join(_strPath1, _strPath2)
return abspath(strNewPath)
@classmethod
def getAbsolutePath(cls, _strPath):
"""
Returns the absolute path.
Deprecated: please use 'os.path.abspath' instead
"""
EDVerbose.log(
"EDUtilsPath.getAbsolutePath is deprecated, please use 'os.path.abspath' instead"
)
return abspath(_strPath)
@classmethod
def existPath(cls, _strPath):
"""
Checks if a folder exists.
Deprecated: please use 'os.path.exists' instead
"""
EDVerbose.log(
"EDUtilsPath.existPath is deprecated, please use 'os.path.exists' instead"
)
return exists(_strPath)
@classmethod
def getEdnaHome(cls):
"""
Returns the EDNA_HOME variable from environment if not already defined
"""
return cls.EDNA_HOME
@classmethod
def setEdnaHome(cls, _strEdnaHome):
"""
Sets EDNA_HOME home
"""
cls.EDNA_HOME = _strEdnaHome
@classmethod
def getEdnaSite(cls):
"""
Returns the EDNA_SITE variable from environment
"""
# EDNA Site
if (cls._EDNA_SITE == None):
cls._EDNA_SITE = os.environ.get("EDNA_SITE")
if (cls._EDNA_SITE is None):
EDVerbose.warning(
"EDUtilsPath.getEdnaSite: EDNA_SITE not set, using EDNA_SITE='default'"
)
cls._EDNA_SITE = "default"
else:
EDVerbose.DEBUG(
"EDUtilsPath.getEdnaSite: EDNA_SITE is set: %s" %
cls._EDNA_SITE)
return cls._EDNA_SITE
@classmethod
def setEdnaSite(cls, _strEdnaSite):
"""
Sets EDNA_SITE
"""
cls._EDNA_SITE = _strEdnaSite
EDNA_SITE = classproperty(getEdnaSite, setEdnaSite)
@classmethod
def getCwd(cls):
"""
Returns the current directory.
Deprecated: please use 'os.getcwd' instead
"""
EDVerbose.log(
"EDUtilsPath.getCwd is deprecated, please use 'os.getcwd' instead")
return os.getcwd()
@classmethod
def createFolder(cls, _strFolderName):
"""
Creates a folder (directory) if it doesn't already exists.
This used to be deprecated but IS neverthless thread safe (see bug#681)
"""
EDVerbose.log("EDUtilsPath.createFolder: %s" % _strFolderName)
with cls.__semaphore:
if (not exists(_strFolderName)):
os.makedirs(_strFolderName)
@classmethod
def getFileList(cls, _strPath="./", _strMask="*"):
"""
Returns a list of file and directory names given a path to a directory.
"""
if _strMask == "*":
listFile = os.listdir(_strPath)
else:
strFind = os.path.join(_strPath, _strMask)
listPath = glob.glob(strFind)
listFile = []
for strPath in listPath:
strPath = os.path.basename(strPath)
listFile.append(strPath)
return listFile
@classmethod
def getFolderName(cls, _str):
"""
Returns the name of a folder (directory) for a given path.
Deprecated: please use 'os.path.dirname' instead
"""
return dirname(_str)
@classmethod
def getEdnaUserTempFolder(cls):
"""
Returns the name of a temporary folder that is unique for a given user.
"""
if cls._EDNA_USERTEMPFOLDER is None:
if os.environ.has_key("EDNA_USERTEMPFOLDER"):
cls._EDNA_USERTEMPFOLDER = os.environ["EDNA_USERTEMPFOLDER"]
if not os.path.exists(cls._EDNA_USERTEMPFOLDER):
EDVerbose.warning(
"EDNA_USERTEMPFOLDER environment variable is set to %s put the directory does not exist!"
% cls._EDNA_USERTEMPFOLDER)
cls._EDNA_USERTEMPFOLDER = None
elif (not os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK)):
EDVerbose.warning(
"EDNA_USERTEMPFOLDER environment variable is set to %s put the directory cannot be accessed!"
% cls._EDNA_USERTEMPFOLDER)
cls._EDNA_USERTEMPFOLDER = None
if cls._EDNA_USERTEMPFOLDER is None:
strEdnaTempFileDir = tempfile.gettempdir()
try:
# Working on Windows and Linux:
strUserName = getpass.getuser()
except Exception:
# Working on MacOS:
strUserName = os.getlogin()
bIsOk = False
# Check that we have write access to this directory:
if os.access(strEdnaTempFileDir, os.W_OK) and os.access(
strEdnaTempFileDir, os.X_OK):
cls._EDNA_USERTEMPFOLDER = os.path.join(
strEdnaTempFileDir, "edna-%s" % strUserName)
# Check that we have write access to this directory:
if not os.path.exists(cls._EDNA_USERTEMPFOLDER):
try:
os.mkdir(cls._EDNA_USERTEMPFOLDER)
except Exception:
EDVerbose.WARNING(
"Error when trying to create the directory %s"
% cls._EDNA_USERTEMPFOLDER)
if os.access(cls._EDNA_USERTEMPFOLDER,
os.W_OK) and os.access(
cls._EDNA_USERTEMPFOLDER, os.X_OK):
bIsOk = True
if not bIsOk:
# We cannot use the edna-<user name> folder...
EDVerbose.WARNING(
"EDUtilsFile.getEdnaUserTempFolder: cannot access user temporary directory %s"
% cls._EDNA_USERTEMPFOLDER)
# Create temporary directory
cls._EDNA_USERTEMPFOLDER = tempfile.mkdtemp(prefix="edna-")
EDVerbose.WARNING(
"Created temporary directory for this session: %s" %
cls._EDNA_USERTEMPFOLDER)
EDVerbose.WARNING(
"If you would like to continue to use this directory for future sessions"
)
EDVerbose.WARNING(
"please set then environment variable EDNA_USERTEMPFOLDER to %s"
% cls._EDNA_USERTEMPFOLDER)
return cls._EDNA_USERTEMPFOLDER
@classmethod
def getEdnaPluginCachePath(cls):
"""This private method initializes the path to the plugin cache file"""
if cls._EDNA_PLUGINCACHE is None:
if "EDNA_PLUGINCACHE" in os.environ.keys():
cls._EDNA_PLUGINCACHE = os.environ["EDNA_PLUGINCACHE"]
strDirName = os.path.dirname(cls._EDNA_PLUGINCACHE)
if os.path.exists(cls._EDNA_PLUGINCACHE) and (not os.access(
cls._EDNA_PLUGINCACHE, os.W_OK)):
EDVerbose.warning(
"EDNA_PLUGINCACHE environment variable is set to %s but the file is not writeable!"
% cls._EDNA_PLUGINCACHE)
cls._EDNA_PLUGINCACHE = None
elif not os.path.exists(strDirName):
EDVerbose.warning(
"EDNA_PLUGINCACHE environment variable is set to %s put the parent directory does not exist!"
% cls._EDNA_PLUGINCACHE)
cls._EDNA_PLUGINCACHE = None
elif not os.access(strDirName, os.W_OK):
EDVerbose.warning(
"EDNA_PLUGINCACHE environment variable is set to %s put the parent directory cannot be accessed!"
% cls._EDNA_PLUGINCACHE)
cls._EDNA_PLUGINCACHE = None
if cls._EDNA_PLUGINCACHE is None:
strTempFileDir = EDUtilsPath.getEdnaUserTempFolder()
# We create a hash of the path in order to be able to reference several different EDNA_HOME
# caches in the same directory
strCacheFileName = hashlib.sha1(
os.getenv('EDNA_HOME')).hexdigest() + ".xml"
cls._EDNA_PLUGINCACHE = os.path.abspath(
os.path.join(strTempFileDir, strCacheFileName))
EDVerbose.DEBUG("EDFactoryPlugin: Path to plugin cache: %s" %
cls._EDNA_PLUGINCACHE)
return cls._EDNA_PLUGINCACHE
@classmethod
def getEdnaTestPath(cls):
"""
This class method initializes the path to the test path (probably unused)
"""
if cls._EDNA_TESTS is None:
if "EDNA_TESTS" in os.environ.keys():
cls.setEdnaTestDataPath(os.environ["EDNA_TESTS"])
if cls._EDNA_TESTS is None:
cls._EDNA_TESTS = os.path.join(
cls.EDNA_HOME,
"tests",
)
EDVerbose.DEBUG(
"EDFactoryPlugin: Path to Test Data directory : %s" %
cls._EDNA_TESTS)
return cls._EDNA_TESTS
@classmethod
def setEdnaTestPath(cls, value):
"""
This class method initializes the path to the test data (probably unused)
"""
cls._EDNA_TESTS = os.path.abspath(value)
if not os.path.isdir(cls._EDNA_TESTS):
EDVerbose.warning(
"EDNA_TESTS environment variable is set to %s put the directory does not exist!"
% cls._EDNA_TESTS)
cls._EDNA_TESTS = None
EDNA_TESTS = classproperty(getEdnaTestPath, setEdnaTestPath)
@classmethod
def getEdnaTestDataPath(cls):
"""
This class method initializes the path to the test data (probably unused)
"""
if cls._EDNA_TESTDATA is None:
if "EDNA_TESTDATA" in os.environ.keys():
cls.setEdnaTestDataPath(os.environ["EDNA_TESTDATA"])
if cls._EDNA_TESTDATA is None:
cls._EDNA_TESTDATA = os.path.join(cls.EDNA_HOME, "tests",
"data")
EDVerbose.DEBUG(
"EDFactoryPlugin: Path to Test Data directory : %s" %
cls._EDNA_TESTDATA)
return cls._EDNA_TESTDATA
@classmethod
def setEdnaTestDataPath(cls, value):
"""
This class method initializes the path to the test data (probably unused)
"""
cls._EDNA_TESTDATA = os.path.abspath(value)
if not os.path.isdir(cls._EDNA_TESTDATA):
EDVerbose.warning(
"EDNA_TESTDATA environment variable is set to %s put the directory does not exist!"
% cls._EDNA_TESTDATA)
cls._EDNA_TESTDATA = None
EDNA_TESTDATA = classproperty(getEdnaTestDataPath, setEdnaTestDataPath)
@classmethod
def getEdnaTestDataImagesPath(cls):
"""
This class method initializes the path to the test images downloaded from internet
"""
if cls._EDNA_TESTIMAGES is None:
if "EDNA_TESTIMAGES" in os.environ.keys():
cls.setEdnaTestDataImagesPath(os.environ["EDNA_TESTIMAGES"])
if cls._EDNA_TESTIMAGES is None:
cls._EDNA_TESTIMAGES = os.path.join(cls.EDNA_HOME, "tests",
"data", "images")
EDVerbose.DEBUG(
"EDFactoryPlugin: Path to Temporary Images directory : %s" %
cls._EDNA_TESTIMAGES)
return cls._EDNA_TESTIMAGES
@classmethod
def setEdnaTestDataImagesPath(cls, value):
"""
This class method initializes the path to the test images downloaded from internet
"""
cls._EDNA_TESTIMAGES = os.path.abspath(value)
if os.path.exists(cls._EDNA_TESTIMAGES) and (not os.access(
cls._EDNA_TESTIMAGES, os.W_OK)):
EDVerbose.warning(
"EDNA_TESTIMAGES environment variable is set to %s but the file is not writeable!"
% cls._EDNA_TESTIMAGES)
cls._EDNA_TESTIMAGES = None
elif not os.path.exists(cls._EDNA_TESTIMAGES):
EDVerbose.warning(
"EDNA_TESTIMAGES environment variable is set to %s put the directory does not exist!"
% cls._EDNA_TESTIMAGES)
cls._EDNA_TESTIMAGES = None
elif not os.access(cls._EDNA_TESTIMAGES, os.W_OK):
EDVerbose.warning(
"EDNA_TESTIMAGES environment variable is set to %s put the directory cannot be accessed!"
% cls._EDNA_TESTIMAGES)
cls._EDNA_TESTIMAGES = None
EDNA_TESTIMAGES = classproperty(getEdnaTestDataImagesPath,
setEdnaTestDataImagesPath)
@classmethod
def getDictOfPaths(cls):
"""
Return an os.environ like dict enriched with internal path
"""
res = {
"${EDNA_HOME}": cls.EDNA_HOME,
"${EDNA_SITE}": cls.EDNA_SITE,
"${EDNA_TESTS}": cls.EDNA_TESTS,
"${EDNA_TESTDATA}": cls.EDNA_TESTDATA,
"${EDNA_TESTIMAGES}": cls.EDNA_TESTIMAGES,
"${USER}": cls._USER,
"${TMPDIR}": cls._TMPDIR
}
return res
class EDObject(object):
"""
Virtual base class for all EDNA Objects (classes).
It offers some synchronization and locking capabilities to make the code thread safe.
"""
__semaphoreId = Semaphore()
__iId_class = 0
profiling = {}
def __init__(self):
"""
Constructor of the main pure virtual class.
This constructor implements:
- the creation of the semaphore
- definition of timer object (uninitialized as potentially not used)
"""
object.__init__(self)
with self.__class__.__semaphoreId:
self.__class__.__iId_class += 1
self.__iId = self.__class__.__iId_class
if self.__class__.__name__ not in self.__class__.profiling:
self.__class__.profiling[self.__class__.__name__] = []
self.__semaphore = Semaphore()
self.__fTimeInit = None
self.__fTimeEnd = None
self.__classname = None
def getId(self):
return self.__iId
def getClassName(self):
"""
Retrieves the name of the class
@return: the name of the class
@rtype: string
"""
return self.__class__.__name__
def synchronizeOn(self):
"""
This method must be used in together with the method synchronizeOff().
This method makes the code thread-safe till the method synchronizeOff
is called.
"""
self.__semaphore.acquire()
def synchronizeOff(self):
"""
This method must be used in together with the method synchronizeOn().
"""
self.__semaphore.release()
def getSemaphoreValue(self):
"""
This method should only be used for debugging purpose...
@return: the "internal" value of the semaphore
@rtype: integer
"""
iValue = self.__semaphore._Semaphore__value
# EDVerbose.WARNING("DEBUG INFO: The value of semaphore for instance of class %s with hash %s is %i" % (self.getClassName(), hash(self), iValue))
return iValue
def locked(self):
return self.__semaphore
def setTimeInit(self):
"""
Initializes the timer for the object
"""
if self.__fTimeInit is None:
self.__fTimeInit = time.time()
def getTimeInit(self):
"""
Retrieves the time of initialization
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeInit
def setTimeEnd(self):
"""
Set the end of calculation time for the given object
"""
if self.__fTimeEnd is None:
self.__fTimeEnd = time.time()
if self.__fTimeInit:
self.__class__.profiling[self.getClassName()].append(self.__fTimeEnd - self.__fTimeInit)
def getTimeEnd(self):
"""
Retrieves the time of end of task
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeEnd
def getRunTime(self):
"""
@returns: the RunTime for the given object
@rtype: float
"""
fRetrunRunTime = 0.0
if self.__fTimeInit is not None:
if self.__fTimeEnd is None:
fRetrunRunTime = time.time() - self.__fTimeInit
else:
fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
return fRetrunRunTime
@classmethod
def analyze_profiling(cls):
"Analyse the profiling an return a list of strings to be printed out"
res = ["Analysis on: %s" % time.asctime(),
" Calls | Mean (s) | Std dev | Total (s) | Plugin name",
"-" * 80]
subres = {}
import numpy
for name, lst_timimgs in cls.profiling.items():
npd = numpy.array(lst_timimgs)
tot = npd.sum()
line = " %6d | %8.3f | %8.3f | %9.3f | %s " % \
(npd.size, tot / npd.size, npd.std(), tot , name)
subres[tot] = line
timimgs = list(subres.keys())
timimgs.sort()
for key in timimgs:
res.append(subres[key])
return res
class CudaCorrelate(object):
plans = {}
data1_gpus = {}
data2_gpus = {}
multconj = None
ctx = None
# pycuda.autoinit.context.pop()
# ctx.pop()
sem = Semaphore()
initsem = Semaphore()
def __init__(self, shape):
self.shape = tuple(shape)
def init(self):
if self.ctx is None:
with self.__class__.initsem:
if self.ctx is None:
self.__class__.ctx = pycuda.autoinit.context
if not self.shape in self.plans:
with self.__class__.initsem:
if not self.shape in self.plans:
self.ctx.push()
if not self.__class__.multconj:
self.__class__.multconj = pycuda.elementwise.ElementwiseKernel(
"pycuda::complex<double> *a, pycuda::complex<double> *b",
"a[i]*=conj(b[i])")
if self.shape not in self.__class__.data1_gpus:
self.__class__.data1_gpus[self.shape] = gpuarray.empty(
self.shape, numpy.complex128)
if self.shape not in self.__class__.data2_gpus:
self.__class__.data2_gpus[self.shape] = gpuarray.empty(
self.shape, numpy.complex128)
if self.shape not in self.__class__.plans:
self.__class__.plans[self.shape] = cu_fft.Plan(
self.shape, numpy.complex128, numpy.complex128)
self.ctx.synchronize()
self.ctx.pop()
@classmethod
def clean(cls):
with initsem:
with sem:
if self.ctx:
cls.ctx.push()
for plan_name in list(cls.plans.keys()):
plan = cls.plans.pop(plan_name)
del plan
for plan_name in list(cls.data1_gpus.keys()):
data = cls.data1_gpus.pop(plan_name)
data.gpudata.free()
del data
for plan_name in cls.data2_gpus.copy():
data = cls.data2_gpus.pop(plan_name)
data.gpudata.free()
del data
cls.ctx.pop()
cls.ctx = None
def correlate(self, data1, data2):
self.init()
with self.__class__.sem:
self.ctx.push()
plan = self.__class__.plans[self.shape]
data1_gpu = self.__class__.data1_gpus[self.shape]
data2_gpu = self.__class__.data2_gpus[self.shape]
data1_gpu.set(data1.astype(numpy.complex128))
cu_fft.fft(data1_gpu, data1_gpu, plan)
data2_gpu.set(data2.astype(numpy.complex128))
cu_fft.fft(data2_gpu, data2_gpu, plan)
# data1_gpu *= data2_gpu.conj()
self.multconj(data1_gpu, data2_gpu)
cu_fft.ifft(data1_gpu, data1_gpu, plan, True)
# self.ctx.synchronize()
res = data1_gpu.get().real
self.ctx.pop()
return res
class EDParallelExecute(EDLogging):
"""
A class helping to make a multi-threaded application from a plugin name and a list of files.
"""
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def runEDNA(self,
_pyListInputPaths=["."],
_strMode="dirwatch",
_bNewerOnly=False):
"""
This method runs the parallel execution on the list of directories.
@param _pyListInputPaths: the name of the directories to look after.
@type _pyListInputPaths: python list of strings
@param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented)
@type _strMode: python string
@param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them.
@type _bNewerOnly: boolean
"""
self.moveToTempDir()
self.__listInputPaths = _pyListInputPaths
if _strMode == "dirwatch":
self.watch_directories(_bNewerOnly)
elif _strMode == "inotify":
print "inotify online notify mode not yet implemented"
raise
else: #mode offline
self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True)
self.waitForAllProcessToFinish()
def moveToTempDir(self):
"""
Create a temporary directory and put all logs there
"""
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-')
self.screen("The log directory of EDNA will be in " +
self.__strTempDir)
os.chdir(self.__strTempDir)
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
jobid = None
if _strXmlInput not in ["", None]:
job = EDJob(self.__strPluginName)
job.setDataInput(_strXmlInput)
job.connectFAILURE(self.failureJobExecution)
job.connectSUCCESS(self.successJobExecution)
job.connectCallBack(self.unregisterJob)
self.semaphoreNbThreadsAcquire()
jobid = job.execute()
self.DEBUG("Running Job id %s" % jobid)
if jobid is None:
self.semaphoreNbThreadsRelease()
return jobid
def successJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with success
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.successJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLout is not None:
job = EDJob.getJobFromID(_jobId)
self.__functXMLout(job.getPlugin().getDataInput(),
job.getPlugin().getDataOutput())
def failureJobExecution(self, _jobId):
"""
Method called when the execution of the plugin finishes with failure
@param _jobId: string of type EDPluginName-number
"""
self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
self.semaphoreNbThreadsRelease()
with self.locked():
if self.__functXMLerr is not None:
self.__functXMLerr(
EDJob.getJobFromID(_jobId).getPlugin().getDataInput())
def unregisterJob(self, _jobid):
"""
remove the filename from the list of files currently under processing
@param _jobId: string of type EDPluginName-number
"""
with self.locked():
for oneKey in self.__dictCurrentlyRunning.copy():
if self.__dictCurrentlyRunning[oneKey] == _jobid:
self.__dictCurrentlyRunning.pop(oneKey)
def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False):
"""
This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode.
@param _listNewFiles: list of files newly created in the directory.
@type _listNewFiles: python list of strings.
@param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online.
@type _bIncludeSubdirs: boolean
"""
for oneFile in _listNewFiles:
if os.path.isdir(oneFile) and _bIncludeSubdirs == True:
for root, _, onesubdirfiles in os.walk(oneFile):
for onesubdirfile in onesubdirfiles:
strFilename = os.path.abspath(
os.path.join(root, onesubdirfile))
if self.__bQuit == True:
return
self.processOneFile(strFilename)
elif os.path.isfile(oneFile):
if self.__bQuit == True:
return
self.processOneFile(oneFile)
def processOneFile(self, _strFilename):
"""
Process on file by calling subsequently the XML generator and the start method unless this file
is already under process (can happend with the watch_directory method).
@param _strFilename: filename to process
@type _strFilename: string
"""
if _strFilename not in self.__dictCurrentlyRunning:
with self.locked():
self.__dictCurrentlyRunning[
_strFilename] = self.__strPluginName
strXmlData = self.__functXMLin(_strFilename)
if strXmlData in [None, ""]:
self.log("Not processing % s" % _strFilename)
with self.locked():
self.__dictCurrentlyRunning.pop(_strFilename)
else:
self.screen("Processing % s" % _strFilename)
jobid = self.start(strXmlData)
with self.locked():
if jobid is None:
self.__dictCurrentlyRunning.pop(_strFilename)
else:
self.__dictCurrentlyRunning[_strFilename] = jobid
def watch_directories(self, _bNewerOnly=False):
"""
Continuously monitors the paths and their subdirectories
for changes. If any files or directories are modified,
the callable function ( here the method self.runEdnaFunction() ) is called
with a list of the modified paths of both
files and directories. This function can return a Boolean value
for rescanning; if it returns True, the directory tree will be
rescanned without calling the function for any found changes.
(This is so this function can write changes into the tree and prevent itself
from being immediately called again.)
@param _bNewerOnly : Do you want to process only newer files
@type _bNewerOnly : Boolean
Basic principle: pyDictAllFiles is a dictionary mapping paths to
modification times. We repeatedly crawl through the directory
tree rooted at 'path', doing a stat() on each file and comparing
the modification time.
"""
dictAllFiles = {}
dictRemainingFiles = {}
listChangedFiles = []
def internalUpdateDict(unused, dirname, files):
"Traversal function for directories"
for strFilename in files:
path = os.path.join(dirname, strFilename)
try:
tempStat = os.stat(path)
except os.error:
# If a file has been deleted between os.path.walk()
# scanning the directory and now, we'll get an
# os.error here. Just ignore it -- we'll report
# the deletion on the next pass through the main loop.
continue
mtime = dictRemainingFiles.get(path)
if mtime is not None:
# Record this file as having been seen
del dictRemainingFiles[path]
# File's mtime has been changed since we last looked at it.
if tempStat.st_mtime > mtime:
listChangedFiles.append(path)
else:
# No recorded modification time, so it must be
# a brand new file.
listChangedFiles.append(path)
# Record current mtime of file.
dictAllFiles[path] = tempStat.st_mtime
if _bNewerOnly:
for path in self.__listInputPaths:
os.path.walk(path, internalUpdateDict, None)
# Main loop
rescan = False
while not self.__bQuit:
listChangedFiles = []
dictRemainingFiles = dictAllFiles.copy()
dictAllFiles = {}
for path in self.__listInputPaths:
os.path.walk(path, internalUpdateDict, None)
#removed_list = dictRemainingFiles.keys()
if rescan:
rescan = False
elif listChangedFiles:
rescan = self.runEdnaFunction(listChangedFiles,
_bIncludeSubdirs=False)
time.sleep(self.__fDelay)
print "Quitting the online mode."
def handleKill(self, signum, frame):
"""
This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program
"""
self.__bQuit = True
sys.stderr.write("Exit requested by signal %s with frame %s.\n" %
(signum, frame))
self.waitForAllProcessToFinish()
os.chdir(self.__strCurrWorkDir)
def flush(self):
"""
This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work
"""
bFinished = False
while not bFinished:
xml = None
try:
xml = self.__functXMLin(None, flush=True)
except TypeError:
try:
xml = self.__functXMLin("", flush=True)
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
xml = None
if (xml is None) or (xml == ""):
bFinished = True
else:
self.screen("Flushing data ...")
self.start(xml)
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
self.screen("Waiting for launched jobs to finish .")
while (self.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
EDJob.stats()
def cleanUp(self, listMethods=[]):
"""
Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD)
@param listMethods: allows to finish some things in the plugin.
@type listMethods: list of strings representing names of methods of the plugin to be called.
"""
self.waitForAllProcessToFinish()
for strOneMethod in listMethods:
try:
print "calling edPlugin.%s" % strOneMethod
exec "edPlugin.%s" % strOneMethod
except Exception:
print "error in processing %s" % strOneMethod
################################################################################
# Nota: there are 2 levels of controls for the number of thread currently running:
# * One here to limit the number of control plugin running at once
# * One on the Exec plugin level for finer grain optimisation
################################################################################
def semaphoreNbThreadsAcquire(self):
"""Method to acquire the semaphore that controls the number of plugins running concurrently"""
# pass
self.__semaphoreNbThreads.acquire()
def semaphoreNbThreadsRelease(self):
"""Method to release the semaphore that controls the number of plugins running concurrently"""
# pass
self.__semaphoreNbThreads.release()
def getNbRunning(self):
"""
Class method:
getter for the number of CPU-active threads running
@return: the number of CPU-active threads runnings
@rtype: integer
"""
#return EDUtilsParallel.getNbRunning()
return self.__iNbThreads - self.__semaphoreNbThreads._Semaphore__value
class EDJob(EDLogging):
"""
Create a module called EDJob
* Most of what was done up to 09-2010 in EDParallelExecute should be done here
* Each instance will be a job
* Constructor takes a plugin name
* Each instance will have taking an "setDataInput" method getting an XMLin (as string)
* Each instance will gave a "getDataOutput" method with optional join
* there could be a "join" method, waiting for the job to finish
* Each instance will have a "execute" method and returning a JobId
* Each instance will have a "setCallBack" method that stores the name of the external callback
* provide status of a job
* keep track of all plugin status
* leave the time to plugin to initialize
* static class retrieve job-instance, status, small-log ...
* prevent multiple run of a single job ?
* does not manage workload of the computer, should be managed at the ExecPlugin level
Used for the tango binding, EDParallelExecute ...
== class variables ==
dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"]
dictJobs [JobId] = EDJob.Instance
== static methods ==
getJob(JobId)
"""
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = Semaphore()
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" %
self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName,
self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
else:
with self.locked():
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
self.WARNING(
"Setting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey).marshal()
elif (self.__pathXSDInput is not None):
return open(self.__pathXSDInput).read()
else:
self.WARNING("Getting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
dataInput = property(getDataInput, setDataInput)
def getDataOutput(self, _strDataOutputKey=None, _bWait=True):
"""
Returns the Plugin Output Data
@param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default.
@type _bWait: boolean
"""
if _bWait: # Wait for plugin to finish befor returning data output
self.synchronize()
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal()
elif self.__pathXSDOutput is not None:
return open(self.__pathXSDOutput).read()
else:
self.WARNING(
"Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ."
% self.__jobId)
dataOutput = property(getDataOutput)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
if not self.__bXmlInputSet:
self.WARNING("Not executing job %s as input is empty" %
self.__jobId)
if (self.__edPlugin is not None):
with self.locked():
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.execute()
return self.__jobId
else:
self.WARNING("Trying to run a plugin that does not exist: %s " %
self.__strPluginName)
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
with self.locked():
strStatus = self.__status
if strStatus == EDJob.PLUGIN_STATE_RUNNING:
self.__edPlugin.synchronize()
elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED:
self.WARNING("Unable to synchronize %s jobs" % strStatus)
else:
self.DEBUG("Unable to synchronize %s jobs" % strStatus)
@classmethod
def synchronizeAll(cls):
"""
Wait for all jobs to finish.
"""
EDVerbose.DEBUG("EDJob.synchronizeAll class method ")
listJob = cls.__dictJobs.keys()
for jobid in listJob:
job = cls.__dictJobs[jobid]
job.synchronize()
if len(cls.__dictJobs) != len(listJob):
EDVerbose.WARNING(
"EDJob.synchronizeAll: New jobs have been launched while synchronizing"
)
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_SUCCESS
self.screen("Plugin %s: success after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Success call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after success) for %s"
% self.__jobId)
self.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_FAILURE
self.screen("Plugin %s: failure after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Failure call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after failure) for %s"
% self.__jobId)
self.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
def connectFAILURE(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
def connectCallBack(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
jobId = property(getJobId, "EDJob.jobId: read-only property")
getJobID = getJobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
pluginName = property(getPluginName,
"EDJob.pluginName: read-only property")
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
plugin = property(getPlugin, "EDJob.plugin: read-only property")
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
status = property(getStatus, "EDJob.status: read-only property")
def getName(self):
return self.__name
def setName(self, _strName):
if self.__name is None:
self.__name = _strName
else:
self.WARNING("EDJob.setName: One cannot rename a Job !!!")
name = property(getName, setName, "EDJob.name: nickname of the job")
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@classmethod
def getStatusFromID(cls, jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in cls.__dictJobs:
strRet = cls.__dictJobs[jobId].getStatus()
else:
strRet = "Unable to retrieve such job: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
getStatusFromId = getStatusFromID
@classmethod
def getJobFromID(cls, jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in cls.__dictJobs:
return cls.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
getJobFromId = getJobFromID
@classmethod
def getMemoryFootprint(cls):
if asizeof is not None:
return asizeof.asizesof(cls.__dictJobs)
def cleanJob(self, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
self.synchronize()
with self.locked():
if self.__edPlugin is not None:
self.__pathXSDOutput = self.__edPlugin.strPathDataOutput
self.__pathXSDInput = self.__edPlugin.strPathDataInput
self.__runtime = self.__edPlugin.getRunTime()
self.__edPlugin = None
if forceGC:
gc.collect()
@classmethod
def cleanJobfromId(cls, jobId, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param jobId: the Job identification number
@type jobId: string
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
if jobId in cls.__dictJobs:
job = cls.__dictJobs[jobId]
job.cleanJob(forceGC)
strRet = "Job %s cleaned" % jobId
else:
strRet = "Unable to retrieve such EDJob: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
cleanJobfromID = cleanJobfromId
@classmethod
def getDataOutputFromId(cls, jobId):
"""
Returns the Plugin Output Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataOutput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataOutput()
return output or ""
getDataOutputFromID = getDataOutputFromId
@classmethod
def getDataInputFromId(cls, jobId):
"""
Returns the Plugin Input Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataInput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataInput()
return output or ""
getDataInputFromID = getDataInputFromId
@classmethod
def countRunning(cls):
"""
return the number of jobs still running.
"""
running = 0
for jobid in list(cls.__dictJobs.keys()): # python3 !
running += (
cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING)
return running
@classmethod
def stats(cls):
"""
Retrieve some statistics and print them
"""
lstStrOut = []
output = []
fExecTime = time.time() - cls.__fStartTime
keys = cls.__dictJobs.keys()
keys.sort()
for num, key in enumerate(keys):
job = cls.__dictJobs[key]
if job.getPlugin() is None:
runtime = job.__runtime
else:
runtime = job.getPlugin().getRunTime()
output.append(
[num, key,
job.getStatus(), runtime,
job.getMemSize()])
output.sort()
iNbJob = max(1, len(keys))
lstStrOut.append("_" * 110)
lstStrOut.append(
"%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
("nr", "EDPluginName-Id", "status", "runtime", "memory"))
lstStrOut.append("_" * 110)
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
tuple(oneJob))
lstStrOut.append("_" * 110)
lstStrOut.append(
"Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
% (fWall, fExecTime, fWall / fExecTime))
lstStrOut.append(
"Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
% (fWall / iNbJob, fExecTime / iNbJob))
if len(keys) > 1:
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
else:
fSlope = 0.0
fOrd = fWall
lstStrOut.append(
"Regression of execution time: ExecTime = %.3f + %f * NbJob" %
(fOrd, fSlope))
strOutput = os.linesep.join(lstStrOut)
EDVerbose.screen(strOutput)
return strOutput
class EDApplication(object):
"""
This is the main EDNA application class. This class can be sub-classed for any specific application need.
An EDNA application is able to launch an entry point plugin. It accepts the following parameter:
--execute : name of the plugin to be executed
--inputFile : related plugin data (xml input data file name)
--outputFile : related plugin result (xml output data file name)
--conf : configuration file name
--basedir : where the application working directory should go
--DEBUG or --debug : turns on debugging
-v or --version : Displays the application name and version
--verbose : Turns on verbose mode
--no-log : Turns off logging
-h or --help : Prints out an usage message
"""
CONFIGURATION_PARAM_LABEL = "--conf"
PLUGIN_PARAM_LABEL = "--execute"
DATASET_PARAM_LABEL = "--inputFile"
OUTPUT_PARAM_LABEL = "--outputFile"
DATASET_BASE_DIRECTORY = "--basedir"
DEBUG_PARAM_LABEL_1 = "--DEBUG"
DEBUG_PARAM_LABEL_2 = "--debug"
VERSION_PARAM_LABEL_1 = "-v"
VERSION_PARAM_LABEL_2 = "--version"
VERBOSE_MODE_LABEL = "--verbose"
NO_LOG_LABEL = "--no-log"
HELP_LABEL_1 = "-h"
HELP_LABEL_2 = "--help"
__edConfiguration = None
__edFactoryPlugin = None
__semaphore = Semaphore()
def __init__(self, _strName="EDApplication", \
_strVersion="1.0.1", \
_strPluginName=None, \
_strConfigurationFileName=None, \
_strDataInputFilePath=None, \
_edLogFile=None, \
_strBaseDir=None, \
_strWorkingDir=None, \
_strDataOutputFilePath=None):
self.__strName = _strName
self.__strVersion = _strVersion
self.__strPluginName = _strPluginName
self.__strConfigurationFileName = _strConfigurationFileName
self.__strDataInputFilePath = _strDataInputFilePath
self.__strDataOutputFilePath = _strDataOutputFilePath
self.__edLogFile = _edLogFile
self.__strBaseDir = _strBaseDir
self.__strWorkingDir = _strWorkingDir
self.__strFullApplicationWorkingDirectory = None
self.__strXMLData = None
self.__listErrorMessages = []
self.__listWarningMessages = []
self.__xsDataOutput = None
self.__edObtainedOutputDataFile = None
self.__strDataOutputFilePath = None
self.__edPlugin = None
self.__edCommandLine = EDCommandLine(sys.argv)
self.__strApplicationInstanceName = self.__strName + "_" + time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
self.__strLogFileName = self.__strApplicationInstanceName + ".log"
self.__bIsFailure = False
self.__strCurrentWorkingDirectory = os.getcwd()
self.__strConfigurationHome = None
self.__strPathToLogFile = None
def execute(self):
"""
This is the main execute method which executes preProcess, process and postProcess.
"""
self.preProcess()
self.process()
self.postProcess()
def preProcess(self):
"""
Creates the application working directory (log dir)
Initializes the configuration
retrieves the plugin xml data to be passed to the plugin
"""
EDVerbose.DEBUG("EDApplication.preProcess")
self.processCommandline()
if (not self.__bIsFailure):
# Check that the plugin can be located
strPluginLocation = EDFactoryPluginStatic.getFactoryPlugin().getModuleLocation(self.__strPluginName)
if (strPluginLocation is None):
EDVerbose.error("Plugin %s cannot be loaded!" % self.__strPluginName)
self.__bIsFailure = True
# Check that the input file can be read
if (self.getDataInputFilePath() is not None) and (not os.path.exists(self.__strDataInputFilePath)):
EDVerbose.error("Input XML file not found : %s" % self.__strDataInputFilePath)
self.__bIsFailure = True
# Check that the output file can be created
if (self.__strDataOutputFilePath is not None):
strOutputDirectory = os.path.dirname(self.__strDataOutputFilePath)
if (strOutputDirectory is None or strOutputDirectory == ""):
strOutputDirectory = os.getcwd()
self.__strDataOutputFilePath = os.path.join(strOutputDirectory, self.__strDataOutputFilePath)
if (not os.access(strOutputDirectory, os.W_OK)):
EDVerbose.error("Output directory not writable: %s" % strOutputDirectory)
self.__bIsFailure = True
elif (os.path.exists(self.__strDataOutputFilePath)):
if (not os.access(self.__strDataOutputFilePath, os.W_OK)):
EDVerbose.error("Output file not writable: %s" % self.__strDataOutputFilePath)
self.__bIsFailure = True
if (not self.__bIsFailure):
EDVerbose.DEBUG("EDApplication.PLUGIN_PARAM_LABEL: " + EDApplication.PLUGIN_PARAM_LABEL)
# Load the configuration file
if (os.path.exists(self.__strConfigurationFileName)):
EDVerbose.screen("Loading Configuration file: %s" % self.__strConfigurationFileName)
EDConfigurationStatic.addConfigurationFile(self.__strConfigurationFileName, _bReplace=True)
pyDictionary = {}
pyDictionary[ "${EDNA_HOME}" ] = EDUtilsPath.getEdnaHome()
if self.getDataInputFilePath() is not None:
self.__strXMLData = EDUtilsFile.readFileAndParseVariables(self.getDataInputFilePath(), pyDictionary)
else:
EDVerbose.warning("Cannot find configuration file: %s" % self.__strConfigurationFileName)
# Create the application working directory
if(self.__strWorkingDir is None):
self.__strWorkingDir = self.__strApplicationInstanceName
self.createApplicationWorkingDirectory()
def process(self):
"""
Calls the Plugin to be executed
"""
if (not self.__bIsFailure):
self.__edPlugin = EDFactoryPluginStatic.loadPlugin(self.__strPluginName)
if(self.__edPlugin is not None):
self.__edPlugin.setBaseDirectory(self.__strFullApplicationWorkingDirectory)
self.__edPlugin.setBaseName(self.__strPluginName)
self.__edPlugin.setDataInput(self.__strXMLData)
self.__edPlugin.connectSUCCESS(self.doSuccessActionPlugin)
self.__edPlugin.connectFAILURE(self.doFailureActionPlugin)
EDVerbose.DEBUG("EDApplication.process: Executing " + self.__strPluginName)
self.__edPlugin.execute()
self.__edPlugin.synchronize()
else:
EDVerbose.error(EDMessage.ERROR_PLUGIN_NOT_LOADED_02 % ('EDApplication.process', self.__strPluginName))
self.__bIsFailure = True
def processCommandline(self):
"""
This method is intended to be overridden by applications who
would like to implement their own command line handling.
This default method implements the following workflow:
- Check for debug, verbose and log file command line options
"""
EDVerbose.DEBUG("EDApplication.execute")
EDVerbose.log(self.__edCommandLine.getCommandLine())
self.processCommandLineDebugVerboseLogFile()
# Determine the base directory
if(self.__strBaseDir is None):
self.processCommandLineBaseDirectory()
# Set the name of the log file
self.__strPathToLogFile = os.path.abspath(os.path.join(self.__strBaseDir, self.__strLogFileName))
EDVerbose.setLogFileName(self.__strPathToLogFile)
self.processCommandLineHelp()
if (not self.__bIsFailure):
self.processCommandLineVersion()
if (not self.__bIsFailure):
# Name of the plugin to be executed
if (self.__strPluginName is None):
self.processCommandLinePluginName()
# Path to the input XML file
if (self.__strDataInputFilePath is None):
self.processCommandLineInputFilePath()
# Path to the output XML file
if(self.__strDataOutputFilePath is None):
self.processCommandLineOutputFilePath()
if (self.__bIsFailure):
self.usage()
if (not self.__bIsFailure):
# If strConfigurationFileName is None, this means that it has not been given to the constructor\
# It has been given by the command line\
if(self.__strConfigurationFileName is None):
self.__strConfigurationFileName = self.getCommandLineArgument(EDApplication.CONFIGURATION_PARAM_LABEL)
def processCommandLineDebugVerboseLogFile(self):
EDVerbose.DEBUG("EDApplication.processCommandLineDebugVerboseLogFile")
EDVerbose.setVerboseOff()
# Check if no log file
if (self.__edCommandLine.existCommand(EDApplication.NO_LOG_LABEL)):
EDVerbose.setLogFileOff()
EDVerbose.DEBUG("Log file output switched off")
# Check if debug mode
if (self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_1) or
self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_2)):
EDVerbose.setVerboseDebugOn()
EDVerbose.DEBUG("Debug Mode [ON]")
# Check if verbose
if (self.__edCommandLine.existCommand(EDApplication.VERBOSE_MODE_LABEL)):
EDVerbose.setVerboseOn()
def processCommandLineHelp(self):
EDVerbose.DEBUG("EDApplication.processCommandLineHelp")
if (self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_1)
or self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_2)):
EDVerbose.setVerboseOn()
self.usage()
self.__bIsFailure = True
def processCommandLineVersion(self):
EDVerbose.DEBUG("EDApplication.processCommandLineVersion")
if (self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_1) or
self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_2)):
EDVerbose.setVerboseOn()
EDVerbose.screen("%s version %s" % (self.__strName, self.__strVersion))
self.__bIsFailure = True
def processCommandLinePluginName(self):
"""
"""
EDVerbose.DEBUG("EDApplication.processCommandLinePluginName")
if (not self.__edCommandLine.existCommand(EDApplication.PLUGIN_PARAM_LABEL)):
EDVerbose.error("No %s command line argument found!" % EDApplication.PLUGIN_PARAM_LABEL)
self.__bIsFailure = True
else:
self.__strPluginName = self.getCommandLineArgument(EDApplication.PLUGIN_PARAM_LABEL)
EDVerbose.DEBUG("EDApplication.processCommandLinePluginName : %s = %s" % (EDApplication.PLUGIN_PARAM_LABEL, self.__strPluginName))
def processCommandLineInputFilePath(self):
"""
"""
EDVerbose.DEBUG("EDApplication.processCommandLineInputFilePath")
if (not self.__edCommandLine.existCommand(EDApplication.DATASET_PARAM_LABEL)):
EDVerbose.error("No %s command line argument found!" % EDApplication.DATASET_PARAM_LABEL)
self.__bIsFailure = True
else:
self.__strDataInputFilePath = self.getCommandLineArgument(EDApplication.DATASET_PARAM_LABEL)
EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.DATASET_PARAM_LABEL, self.__strDataInputFilePath))
def processCommandLineOutputFilePath(self):
"""
"""
EDVerbose.DEBUG("EDApplication.processCommandLineOutputFilePath")
if (not self.__edCommandLine.existCommand(EDApplication.OUTPUT_PARAM_LABEL)):
EDVerbose.DEBUG("No %s command line argument found" % EDApplication.OUTPUT_PARAM_LABEL)
else:
self.__strDataOutputFilePath = self.getCommandLineArgument(EDApplication.OUTPUT_PARAM_LABEL)
EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.OUTPUT_PARAM_LABEL, self.__strDataOutputFilePath))
def processCommandLineBaseDirectory(self):
"""
"""
EDVerbose.DEBUG("EDApplication.processCommandLineBaseDirectory")
self.__strBaseDir = self.getCommandLineArgument(EDApplication.DATASET_BASE_DIRECTORY)
if(self.__strBaseDir is None):
self.__strBaseDir = os.getcwd()
EDVerbose.DEBUG("Base directory set to current working directory = %s" % (self.__strBaseDir))
else:
EDVerbose.DEBUG("%s = %s" % (EDApplication.DATASET_BASE_DIRECTORY, self.__strBaseDir))
def postProcess(self):
"""
"""
# Restore the current working directory
os.chdir(self.__strCurrentWorkingDirectory)
@classmethod
def usage(cls):
"""
Print usage...
"""
EDVerbose.screen("")
EDVerbose.screen("Usage: ")
EDVerbose.screen("")
EDVerbose.screen("%35s : Name of the plugin to be executed" % (cls.PLUGIN_PARAM_LABEL))
EDVerbose.screen("")
EDVerbose.screen("%35s : Path to the XML input file" % (cls.DATASET_PARAM_LABEL))
EDVerbose.screen("")
EDVerbose.screen("-----------------------------------------------------------------------------------------------------------")
EDVerbose.screen("")
EDVerbose.screen(" Additional options available:")
EDVerbose.screen("")
EDVerbose.screen("%35s : Path to the file wich will contain the XML output" % (cls.OUTPUT_PARAM_LABEL))
EDVerbose.screen("")
EDVerbose.screen("%35s : Base directory, i.e. working directory for the application" % (cls.DATASET_BASE_DIRECTORY))
EDVerbose.screen("")
EDVerbose.screen("%35s : Verbose mode" % (cls.VERBOSE_MODE_LABEL))
EDVerbose.screen("")
EDVerbose.screen("%35s : XSConfiguration file" % (cls.CONFIGURATION_PARAM_LABEL))
EDVerbose.screen("")
EDVerbose.screen("%35s : Executable version info" % (cls.VERSION_PARAM_LABEL_1 + " or " + cls.VERSION_PARAM_LABEL_2))
EDVerbose.screen("")
EDVerbose.screen("%35s : DEBUG log traces" % (cls.DEBUG_PARAM_LABEL_1 + " or " + cls.DEBUG_PARAM_LABEL_2))
EDVerbose.screen("")
EDVerbose.screen("%35s : No log file" % (cls.NO_LOG_LABEL))
EDVerbose.screen("")
EDVerbose.screen("%35s : This help message" % (cls.HELP_LABEL_1 + " or " + cls.HELP_LABEL_2))
EDVerbose.screen("")
@classmethod
def getFactoryPlugin(cls):
EDVerbose.WARNING("the use of EDclsetFactoryPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
return EDFactoryPluginStatic.getFactoryPlugin
@classmethod
def loadPlugin(cls, _strPluginName):
EDVerbose.WARNING("The use of EDApplication.loadPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
return EDFactoryPluginStatic.loadPlugin(_strPluginName)
@classmethod
def loadModule(cls, _strModuleName):
EDVerbose.WARNING("The use of EDApplication.loadModule is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
EDFactoryPluginStatic.loadModule(_strModuleName)
def getDataInputFilePath(self):
return self.__strDataInputFilePath
def getBaseDir(self):
"""
Getter for base directory
@return: path of the base directory
@rtype: string
"""
return self.__strBaseDir
def createApplicationWorkingDirectory(self):
"""
Created the working directory of the application (<date>-<application name>)
First tries to retrieve the base dir from --basedir option or related parameter from constructor
Otherwise tries to retrieve it from EDNA_BASE_DIRECTORY environment variable
Otherwise put the base dir as the current directory
"""
EDVerbose.DEBUG("EDApplication.createApplicationWorkingDirectory")
strBaseDirectory = self.getBaseDir()
strDateTime = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory, self.__strWorkingDir)
# Check that a folder / file with the same name already exists
if(os.path.exists(self.__strFullApplicationWorkingDirectory) or \
os.path.exists(self.__strFullApplicationWorkingDirectory)):
# It does exist so we have to modify the name of the working directory
iIndex = 1
bContinueFlag = True
while (bContinueFlag):
self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory,
"%s_%d" % \
(strDateTime, \
iIndex))
if(os.path.isdir(self.__strFullApplicationWorkingDirectory) or \
os.path.exists(self.__strFullApplicationWorkingDirectory)):
iIndex += 1
else:
bContinueFlag = False
# Make the directory
os.mkdir(self.__strFullApplicationWorkingDirectory)
# Change it to be the current working directory
os.chdir(self.__strFullApplicationWorkingDirectory)
def getFullApplicationWorkingDirectory(self):
return self.__strFullApplicationWorkingDirectory
def getCurrentWorkingDirectory(self):
return self.__strCurrentWorkingDirectory
def doSuccessActionPlugin(self, _edPlugin):
"""
"""
EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin")
# Print the potential Warnings and Errors
self.__listWarningMessages = _edPlugin.getListOfWarningMessages()
EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Warnings " % (_edPlugin.getPluginName(), len(self.__listWarningMessages)))
for warningMessage in self.__listWarningMessages:
EDVerbose.screen(warningMessage)
self.__listErrorMessages = _edPlugin.getListOfErrorMessages()
EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Errors" % (_edPlugin.getPluginName(), len(self.__listErrorMessages)))
for errorMessage in self.__listErrorMessages:
EDVerbose.error(errorMessage)
if (_edPlugin.hasDataOutput()):
xsDataOutput = _edPlugin.getDataOutput()
if (xsDataOutput is not None and self.__strDataOutputFilePath is not None):
xsDataOutput.exportToFile(self.__strDataOutputFilePath)
if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None):
xsDataOutput.exportToFile(self.__edObtainedOutputDataFile)
def doFailureActionPlugin(self, _edPlugin):
EDVerbose.DEBUG("EDApplication.doFailureActionPlugin")
# Print the potential Warnings and Errors
EDVerbose.DEBUG("EDApplication.doFailureActionPlugin: Plugin %s failed" % _edPlugin.getClassName())
self.__listWarningMessages = _edPlugin.getListOfWarningMessages()
for warningMessage in self.__listWarningMessages:
EDVerbose.screen(warningMessage)
self.__listErrorMessages = _edPlugin.getListOfErrorMessages()
for errorMessage in self.__listErrorMessages:
EDVerbose.screen(errorMessage)
if (_edPlugin.hasDataOutput()):
xsDataOutput = _edPlugin.getDataOutput()
if (xsDataOutput is not None and self.__strDataOutputFilePath is not None):
xsDataOutput.exportToFile(self.__strDataOutputFilePath)
if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None):
xsDataOutput.exportToFile(self.__edObtainedOutputDataFile)
def getPlugin(self):
return self.__edPlugin
def getPluginOutputData(self):
return self.__xsDataOutput
def getWarningMessages(self):
return self.__listWarningMessages
def getErrorMessages(self):
return self.__listErrorMessages
def getEdCommandLine(self):
return self.__edCommandLine
def getCommandLine(self):
return self.__edCommandLine.getCommandLine()
def getCommandLineArguments(self):
with self.__class__.__semaphore:
edCommandLine = self.__edCommandLine.getCommandLine()
return edCommandLine
def getCommandLineArgument(self, _strKey):
with self.__class__.__semaphore:
strCommandLineArgument = self.__edCommandLine.getArgument(_strKey)
return strCommandLineArgument
@classmethod
def synchronizeOn(cls):
"""
Lock the whole class
"""
cls.__semaphore.acquire()
@classmethod
def synchronizeOff(cls):
"""
Unlock the whole class
"""
cls.__semaphore.release()
def getApplicationName(self):
return self.__strName + "-" + self.__strVersion
def getWorkingDir(self):
"""
Getter for working dir
@rtype: string
@return working dir
"""
return self.__strWorkingDir
def setWorkingDir(self, _strDir):
"""
Setter for working dir
@type _strDir: string
@param _strDir: working dir
"""
self.__strWorkingDir = _strDir
def isFailure(self):
return self.__bIsFailure
def setFailure(self, _bFailure):
self.__bIsFailure = _bFailure
def getPluginName(self):
return self.__strPluginName
class EDPluginBioSaxsSmartMergev1_7(EDPluginControl):
"""
This plugin takes a set of input data files (1D SAXS) measure
their differences (versus previous and versus first) and merge those which are equivalent
v1.1 adds information from AutoSub
Controlled plugins:
- Execplugins/WaitMultifile
- EdnaSaxs/Atsas/DatAver
- EdnaSaxs/Atsas/AutoSub
- EdnaSaxs/SaxsAnalysis
"""
# dictAve = {} #key=?; value=path to average file
lastBuffer = None
lastSample = None
dictFrames = {
} #dictionary key = filename of average, value = list of files used for average
__strControlledPluginDataver = "EDPluginExecDataverv2_0"
#__strControlledPluginDatcmp = "EDPluginExecDatcmpv2_0"
__strControlledPluginWaitFile = "EDPluginWaitMultiFile"
__strControlledPluginAutoSub = "EDPluginAutoSubv1_1"
__strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0"
__strControlledPluginSaxsISPyB = "EDPluginBioSaxsISPyBv1_0"
__configured = False
CONF_MINIMUM_CURVE_FILE_SIZE = "MinCurveFileSize"
minimumCurveFileSize = 10000
semaphore = Semaphore()
cpRsync = "EDPluginExecRsync"
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
#self.__edPluginExecDatcmp = None
self.__edPluginExecDataver = None
self.__edPluginExecWaitFile = None
self.__edPluginExecAutoSub = None
self.__edPluginSaxsAnalysis = None
self.__edPluginSaxsISPyB = None
self.setXSDataInputClass(XSDataInputBioSaxsSmartMergev1_0)
#self.__edPluginExecDatCmp = None
self.lstInput = []
self.curves = []
self.forgetLastSample = False
self.lstMerged = []
self.lstDiscarded = []
self.lstXsdInput = []
self.absoluteFidelity = 0.01 #None
self.relativeFidelity = 0.01 #None
self.dictSimilarities = {} # key: 2-tuple of images, similarities
self.lstStrInput = []
self.autoRg = None
self.gnom = None
self.volume = None
self.rti = None
self.strRadiationDamage = None
self.strMergedFile = None
self.lstSub = []
self.strSubFile = None
self.fConcentration = None
self.xsDataResult = XSDataResultBioSaxsSmartMergev1_0()
self.xsBestBuffer = None
self.bestBufferType = ""
self.bufferFrames = []
self.xsScatterPlot = None
self.xsGuinierPlot = None
self.xsKratkyPlot = None
self.xsDensityPlot = None
self.xsdSubtractedCurve = None
self.outdir = None #directory on rnice for analysis results to go to
def configure(self):
"""
Configures the plugin from the configuration file with the following parameters:
- curve_file_size: minimum size of the file.
"""
EDPluginControl.configure(self)
if not self.__configured:
with self.semaphore:
if not self.__configured:
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.configure")
min_size = self.config.get(
self.CONF_MINIMUM_CURVE_FILE_SIZE)
if min_size is None:
strMessage = 'EDPluginBioSaxsSmartMergev1_7.configure: %s Configuration parameter missing: \
%s, defaulting to "%s"' % (self.getBaseName(),
self.CONF_MINIMUM_CURVE_FILE_SIZE,
self.minimumCurveFileSize)
self.WARNING(strMessage)
self.addErrorWarningMessagesToExecutiveSummary(
strMessage)
else:
self.__class__.minimumCurveFileSize = float(min_size)
self.__class__.__configured = True
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
self.checkMandatoryParameters(self.dataInput.inputCurves,
"Input curve list is empty")
self.checkMandatoryParameters(self.dataInput.mergedCurve,
"Output curve filename is empty")
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.preProcess")
# Load the execution plugin
if self.dataInput.absoluteFidelity is not None:
self.absoluteFidelity = self.dataInput.absoluteFidelity.value
if self.absoluteFidelity == 0.0:
self.absoluteFidelity = None
if self.dataInput.relativeFidelity is not None:
self.relativeFidelity = self.dataInput.relativeFidelity.value
if self.relativeFidelity == 0.0:
self.relativeFidelity = None
self.lstInput = self.dataInput.inputCurves
self.lstStrInput = [i.path.value for i in self.lstInput]
self.__edPluginExecWaitFile = self.loadPlugin(
self.__strControlledPluginWaitFile)
if self.dataInput.mergedCurve is not None:
self.strMergedFile = self.dataInput.mergedCurve.path.value
if self.dataInput.subtractedCurve is not None:
self.strSubFile = self.dataInput.subtractedCurve.path.value
dirname = os.path.dirname(self.strSubFile)
if not os.path.isdir(dirname):
os.mkdir(dirname)
def getFramesByFilename(self, filename):
if filename is not None:
if filename in self.__class__.dictFrames.keys():
return self.__class__.dictFrames[filename]
return {"averaged": [], "discarded": []}
def getAveragedFrameByFilename(self, filename):
frames = self.getFramesByFilename(filename)
if frames is not None:
return frames["averaged"]
return []
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(self.minimumCurveFileSize),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
self.addExecutiveSummaryLine(
"Got only one frame ... nothing to merge !!!")
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
self.datcmp(self.lstInput[0], oneFile)
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
self.datcmp(self.lstInput[0], self.lstInput[1])
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
self.datcmp(self.lstInput[idx + 1], oneFile)
# edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
# edPluginExecRelativeFidelity.setDataInput(xsd)
# edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.addExecutiveSummaryLine("WARNING: " +
self.strRadiationDamage)
self.addExecutiveSummaryLine("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
# outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
self.xsdSubtractedCurve = XSDataFile(XSDataString(sub))
#self.curves.append(xsdSubtractedCurve)
self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer, self.dataInput.mergedCurve
],
subtractedCurve=self.xsdSubtractedCurve)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
if self.isFailure():
return
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.__strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
scatterCurve=self.xsdSubtractedCurve,
autoRg=self.autoRg,
graphFormat=XSDataString("png"))
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
#self.__class__.lastSample = None #Information neededfor transfer to ISPyB
self.forgetLastSample = True
else:
self.__class__.lastSample = self.dataInput.mergedCurve
if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
self.addExecutiveSummaryLine("Registering to ISPyB")
self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged))
self.__class__.dictFrames[self.dataInput.mergedCurve] = {
'averaged': self.lstMerged,
'discarded': self.lstDiscarded
}
self.__edPluginSaxsISPyB = self.loadPlugin(
self.__strControlledPluginSaxsISPyB)
if len(self.lstInput) > 1:
frameAverage = XSDataInteger(len(self.lstInput))
frameMerged = XSDataInteger(len(self.lstMerged))
else:
frameMerged = frameAverage = XSDataInteger(1)
self.curves = [XSDataFile(i.path) for i in self.lstInput]
self.discardedCurves = [
XSDataFile(i.path) for i in self.lstDiscarded
]
self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged]
averageFilePath = None
if self.strMergedFile is not None:
averageFilePath = XSDataFile(XSDataString(self.strMergedFile))
self.sampleFrames = self.getAveragedFrameByFilename(
self.__class__.lastSample)
lastSample = None
if self.__class__.lastSample is not None:
lastSample = self.__class__.lastSample
subtractedCurve = None
if self.xsdSubtractedCurve is not None:
subtractedCurve = self.xsdSubtractedCurve
else:
subtractedCurve = None
xsdin = XSDataInputBioSaxsISPyBv1_0(
sample=self.dataInput.sample,
autoRg=self.autoRg,
gnom=self.gnom,
volume=self.volume,
frameAverage=frameAverage,
frameMerged=frameMerged,
curves=self.curves,
discardedFrames=self.discardedCurves,
averagedFrames=self.mergedCurves,
averageFilePath=averageFilePath,
bufferFrames=self.bufferFrames,
sampleFrames=self.sampleFrames,
bestBuffer=self.xsBestBuffer,
averageSample=lastSample,
scatterPlot=self.xsScatterPlot,
guinierPlot=self.xsGuinierPlot,
kratkyPlot=self.xsKratkyPlot,
densityPlot=self.xsDensityPlot,
subtractedFilePath=subtractedCurve
# destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
)
self.__edPluginSaxsISPyB.dataInput = xsdin
self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
self.__edPluginSaxsISPyB.execute()
# transfer analysis data to correct location on nice
if self.gnom is not None:
self.outdir = os.path.join(
os.path.dirname(os.path.dirname(self.lstStrInput[0])),
"ednaAnalysis")
basename = os.path.basename(
os.path.splitext(self.gnom.gnomFile.path.value)[0])
self.outdir = os.path.join(self.outdir, basename)
if not os.path.isdir(self.outdir):
os.makedirs(self.outdir)
#self.outFile = os.path.join(outdir, "NoResults.html")
workingdir = os.path.dirname(self.gnom.gnomFile.path.value)
self.pluginRsync = self.loadPlugin(self.cpRsync)
self.pluginRsync.dataInput = XSDataInputRsync(
source=XSDataFile(XSDataString(workingdir)),
destination=XSDataFile(XSDataString(self.outdir)),
options=XSDataString("-avx"))
self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync)
self.pluginRsync.connectFAILURE(self.doFailureExecRsync)
self.pluginRsync.executeSynchronous()
if self.forgetLastSample:
#Also redefine dictionary to contain the buffer just processed?
self.__class__.lastSample = None
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.postProcess")
# Create some output data
self.xsDataResult.mergedCurve = self.dataInput.mergedCurve
if self.strSubFile is not None and os.path.isfile(self.strSubFile):
self.xsDataResult.subtractedCurve = XSDataFile(
XSDataString(self.strSubFile))
self.xsDataResult.autoRg = self.autoRg
self.xsDataResult.gnom = self.gnom
self.xsDataResult.volume = self.volume
self.xsDataResult.rti = self.rti
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
self.xsDataResult.status = XSDataStatus(
message=self.getXSDataMessage(),
executiveSummary=XSDataString(
os.linesep.join(self.getListExecutiveSummaryLines())))
self.dataOutput = self.xsDataResult
def resynchronize(self):
"""
Sometimes plugins are not started or not yet finished...
"""
time.sleep(1) # this is just a way to give the focus to other threads
self.synchronizePlugins()
self.ERROR("I slept a second, waiting for sub-plugins to finish")
def rewriteHeader(self,
infile=None,
output=None,
hdr="#",
linesep=os.linesep):
"""
Write the output file with merged data with the good header.
# BSA sample
#Sample c= 0.0 mg/ml
#
# Merged from: file1
# Merged from: file2
# Merged from: file4
#
# Sample Information:
# Concentration: 0.0
# Code: BSA
@param infile: path of the original data file where original data are taken from
@param outfile: path of the destination file (by default, from XSD)
@param hdr: header marker, often "#"
@param linesep: line separator, usually "\n" or "\r\n" depending on the Operating System
@return: None
"""
Code = Concentration = None
frameMax = exposureTime = measurementTemperature = storageTemperature = None
originalFile = self.lstMerged[0].path.value
headers = [
line.strip() for line in open(originalFile) if line.startswith("#")
]
Comments = headers[0][1:].strip()
for line in headers:
if "title =" in line:
Comments = line.split("=", 1)[1].strip()
elif "Comments =" in line:
Comments = line.split("=", 1)[1].strip()
elif "Concentration:" in line:
Concentration = line.split(":", 1)[1].strip()
elif "Concentration =" in line:
Concentration = line.split("=", 1)[1].strip()
elif "Code =" in line:
Code = line.split("=", 1)[1].strip()
elif "Code:" in line:
Code = line.split(":", 1)[1].strip()
elif "Storage Temperature" in line:
storageTemperature = line.split(":", 1)[1].strip()
elif "Measurement Temperature" in line:
measurementTemperature = line.split(":", 1)[1].strip()
elif "Time per frame" in line:
exposureTime = line.split("=", 1)[1].strip()
elif "Frame" in line:
frameMax = line.split()[-1]
try:
c = float(Concentration)
except Exception:
c = -1.0
self.fConcentration = c
lstHeader = [
"%s %s" % (hdr, Comments),
"%s Sample c= %s mg/ml" % (hdr, Concentration), hdr
]
if frameMax is not None:
lstHeader.append(hdr +
" Number of frames collected: %s" % frameMax)
if exposureTime is not None:
lstHeader.append(hdr +
" Exposure time per frame: %s" % exposureTime)
if self.strRadiationDamage is None:
lstHeader.append(
"%s No significant radiation damage detected, merging %i files"
% (hdr, len(self.lstMerged)))
else:
lstHeader.append("%s WARNING: %s" % (hdr, self.strRadiationDamage))
lstHeader += [
hdr + " Merged from: %s" % i.path.value for i in self.lstMerged
]
if self.lstSub:
lstHeader.append(hdr)
lstHeader += ["%s %s" % (hdr, i) for i in self.lstSub]
lstHeader += [hdr, hdr + " Sample Information:"]
if storageTemperature is not None:
lstHeader.append(hdr + " Storage Temperature (degrees C): %s" %
storageTemperature)
if measurementTemperature is not None:
lstHeader.append(hdr + " Measurement Temperature (degrees C): %s" %
measurementTemperature)
lstHeader += [
hdr + " Concentration: %s" % Concentration,
"# Code: %s" % Code
]
if infile is None:
infile = self.strMergedFile
if output is None:
output = self.strMergedFile
data = linesep.join(
[i.strip() for i in open(infile) if not i.startswith(hdr)])
with open(output, "w") as outfile:
outfile.write(linesep.join(lstHeader))
if not data.startswith(linesep):
outfile.write(linesep)
outfile.write(data)
def doSuccessExecWait(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecWait")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait")
self.retrieveMessages(_edPlugin)
xsdo = _edPlugin.dataOutput
self.DEBUG("ExecWait Output:%s" % xsdo.marshal())
if (xsdo.timedOut is not None) and bool(xsdo.timedOut.value):
strErr = "Error in waiting for all input files to arrive"
self.addExecutiveSummaryLine(
"EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait :" + strErr)
self.ERROR(strErr)
self.setFailure()
def doFailureExecWait(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecWait")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecWait")
self.retrieveMessages(_edPlugin)
strErr = "Error in waiting for all input files to arrive"
self.ERROR(strErr)
self.addExecutiveSummaryLine(strErr)
self.setFailure()
def doSuccessExecDataver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver")
self.retrieveMessages(_edPlugin)
self.rewriteHeader(_edPlugin.dataOutput.outputCurve.path.value,
output=self.strMergedFile)
def doFailureExecDataver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDataver")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDataver")
self.retrieveMessages(_edPlugin)
strErr = "Error in Processing of Atsas 'dataver'"
self.addExecutiveSummaryLine(strErr)
self.ERROR(strErr)
self.setFailure()
def datcmp(self, file1, file2):
#def doSuccessExecDatcmp(self, _edPlugin=None):
# self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp")
# self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp")
# self.retrieveMessages(_edPlugin)
data1 = loadtxt(file1.path.value)
data2 = loadtxt(file2.path.value)
with self.locked():
try:
cor = gof(data1, data2)
fidelity = cor.P
except Exception as err:
strErr = "No fidelity in output of datcmp"
self.error(strErr)
self.addExecutiveSummaryLine(strErr)
self.addExecutiveSummaryLine(str(err))
#self.lstExecutiveSummary.append(strErr)
#self.lstExecutiveSummary.append(err)
if not "maximum recursion depth exceeded" in str(err):
self.setFailure()
fidelity = 0
lstIdx = [
self.lstStrInput.index(file1.path.value),
self.lstStrInput.index(file2.path.value)
]
lstIdx.sort()
self.dictSimilarities[tuple(lstIdx)] = fidelity
lstIdx.reverse()
self.dictSimilarities[tuple(lstIdx)] = fidelity
if fidelity <= 0:
logFid = "infinity"
else:
logFid = "%.2f" % (-log(fidelity, 10))
self.addExecutiveSummaryLine(
"-log(Fidelity) between %s and %s is %s" %
(os.path.basename(file1.path.value),
os.path.basename(file2.path.value), logFid))
# def doFailureExecDatcmp(self, _edPlugin=None):
# self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDatcmp")
# self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDatcmp")
# self.retrieveMessages(_edPlugin)
# self.addExecutiveSummaryLine("Failure in Processing of Atsas 'datcmp'")
# self.setFailure()
def doSuccessExecAutoSub(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub")
self.retrieveMessages(_edPlugin)
self.autoRg = _edPlugin.dataOutput.autoRg
if _edPlugin.dataOutput.subtractedCurve is not None:
subcurve = _edPlugin.dataOutput.subtractedCurve
if os.path.exists(subcurve.path.value):
self.strSubFile = subcurve.path.value
self.xsBestBuffer = _edPlugin.dataOutput.bestBuffer
self.bestBufferType = _edPlugin.dataOutput.bestBufferType.value
if self.bestBufferType == 'average':
self.bufferFrames = self.lstMerged
#if self.__class__.dictFrames[self.__class__.lastBuffer] is not None:
# if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None:
# self.bufferFrames = self.bufferFrames + self.__class__.dictFrames[self.__class__.lastBuffer]['averaged']
self.bufferFrames = self.bufferFrames + self.getAveragedFrameByFilename(
self.__class__.lastBuffer)
elif self.bestBufferType == self.__class__.lastBuffer:
#if self.__class__.dictFrames[self.__class__.lastBuffer] is not None:
# if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None:
# self.bufferFrames = self.__class__.dictFrames[self.__class__.lastBuffer]['averaged']
self.bufferFrames = self.getAveragedFrameByFilename(
self.__class__.lastBuffer)
else:
self.bufferFrames = self.lstMerged
self.addExecutiveSummaryLine(
_edPlugin.dataOutput.status.executiveSummary.value)
def doFailureExecAutoSub(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub")
self.retrieveMessages(_edPlugin)
strErr = "Error in Processing of EDNA 'AutoSub'"
# clean up "EDNA memory"
self.__class__.lastBuffer = self.dataInput.mergedCurve
self.__class__.lastSample = None
self.ERROR(strErr)
self.addExecutiveSummaryLine(strErr)
self.setFailure()
def doSuccessSaxsAnalysis(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis")
self.retrieveMessages(_edPlugin)
self.gnom = _edPlugin.dataOutput.gnom
self.volume = _edPlugin.dataOutput.volume
self.rti = _edPlugin.dataOutput.rti
self.xsScatterPlot = _edPlugin.dataOutput.scatterPlot
self.xsGuinierPlot = _edPlugin.dataOutput.guinierPlot
self.xsKratkyPlot = _edPlugin.dataOutput.kratkyPlot
if hasattr(_edPlugin.dataOutput, 'kratkyRgPlot'):
self.xsKratkyRgPlot = _edPlugin.dataOutput.kratkyRgPlot
if hasattr(_edPlugin.dataOutput, 'kratkyVcPlot'):
self.xsKratkyVcPlot = _edPlugin.dataOutput.kratkyVcPlot
self.xsDensityPlot = _edPlugin.dataOutput.densityPlot
self.addExecutiveSummaryLine(
_edPlugin.dataOutput.status.executiveSummary.value)
def doFailureSaxsAnalysis(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureSaxsAnalysis")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureSaxsAnalysis")
self.retrieveMessages(_edPlugin)
strErr = "Error in Processing of EDNA SaxsAnalysis = AutoRg => datGnom => datPorod"
self.ERROR(strErr)
self.addExecutiveSummaryLine(strErr)
self.setFailure()
def doSuccessISPyB(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessISPyB")
self.addExecutiveSummaryLine("Registered in ISPyB")
self.retrieveMessages(_edPlugin)
def doFailureISPyB(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureISPyB")
self.addExecutiveSummaryLine("Failed to registered in ISPyB")
self.retrieveMessages(_edPlugin)
def doSuccessExecRsync(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync")
self.retrieveMessages(_edPlugin)
self.gnom.gnomFile = XSDataFile(
XSDataString(
os.path.join(self.outdir,
os.path.split(self.gnom.gnomFile.path.value)[1])))
def doFailureExecRsync(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync")
self.retrieveMessages(_edPlugin)
self.setFailure()
class EDPluginBioSaxsHPLCv1_0(EDPluginControl):
"""
plugin for processing Saxs data coming from HPLC
runs subsequently:
*ProcessOneFile,
*subtraction of buffer
*SaxsAnalysis
todo:
only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs
"""
strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_3"
strControlledPluginDatop = "EDPluginExecDatopv1_0"
strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0"
strControlledPluginDatCmp = "EDPluginExecDatcmpv1_0"
strControlledPluginDatAver = "EDPluginExecDataverv1_0"
dictHPLC = {} #key=runId, value= HPLCrun instance
_sem = Semaphore()
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0)
self.__edPluginProcessOneFile = None
self.__edPluginSubtract = None
self.__edPluginSaxsAnalysis = None
self.__edPluginDatCmp = None
self.xsDataResult = XSDataResultBioSaxsHPLCv1_0()
self.runId = None
self.frameId = None
self.frame = None
self.hplc_run = None
self.curve = None
self.subtracted = None
self.lstExecutiveSummary = []
self.isBuffer = False
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginBioSaxsHPLCv1_0.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image")
self.checkMandatoryParameters(self.dataInput.sample,
"no Sample parameter")
self.checkMandatoryParameters(self.dataInput.experimentSetup,
"No experimental setup parameter")
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_0.preProcess")
sdi = self.dataInput
if sdi.runId is not None:
self.runId = sdi.runId.value
else:
path = sdi.rawImage.path.value
if "_" in path:
self.runId = path[::-1].split("_", 1)[1][::-1]
else:
self.runId = path
with self._sem:
if self.runId not in self.dictHPLC:
self.dictHPLC[self.runId] = HPLCrun(self.runId)
self.hplc_run = self.dictHPLC[self.runId]
if sdi.frameId is not None:
self.frameId = sdi.frameId.value
else:
path = sdi.rawImage.path.value
if "_" in path:
digits = os.path.splitext(
os.path.basename(path))[0].split("_")[-1]
try:
self.frameId = int(digits)
except ValueError:
self.WARNING(
"frameId is supposed to be an integer, I got %s" %
digits)
self.frameId = digits
else:
self.warning("using frameID=0 in tests, only")
self.frameId = 0
with self._sem:
self.frame = HPLCframe(self.runId, self.frameId)
self.hplc_run.frames[self.frameId] = self.frame
if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value):
with self._sem:
self.hplc_run.buffer = sdi.bufferCurve.path.value
if self.hplc_run.hdf5_filename:
hplc = self.hplc_run.hdf5_filename
elif sdi.hplcFile:
hplc = sdi.hplcFile.path.value
else:
path = sdi.rawImage.path.value
if "_" in path:
hplc = "_".join(
os.path.splitext(path)[0].split("_")[:-1]) + ".h5"
else:
hplc = os.path.splitext(path)[0] + ".h5"
if not self.hplc_run.hdf5_filename:
with self._sem:
self.hplc_run.init_hdf5(hplc)
# self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc))
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_0.process")
xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(
rawImage=self.dataInput.rawImage,
sample=self.dataInput.sample,
experimentSetup=self.dataInput.experimentSetup,
rawImageSize=self.dataInput.rawImageSize,
normalizedImage=self.dataInput.normalizedImage,
integratedCurve=self.dataInput.integratedCurve,
runId=self.dataInput.runId,
frameId=self.dataInput.frameId)
self.__edPluginProcessOneFile = self.loadPlugin(
self.strControlledPluginProcessOneFile)
self.__edPluginProcessOneFile.dataInput = xsdIn
self.__edPluginProcessOneFile.connectSUCCESS(
self.doSuccessProcessOneFile)
self.__edPluginProcessOneFile.connectFAILURE(
self.doFailureProcessOneFile)
self.__edPluginProcessOneFile.executeSynchronous()
if self.isFailure():
return
xsdIn = XSDataInputDatcmp(inputCurve=[
XSDataFile(XSDataString(self.hplc_run.first_curve)),
XSDataFile(XSDataString(self.curve))
])
self.__edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
self.__edPluginDatCmp.dataInput = xsdIn
self.__edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
self.__edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
self.__edPluginDatCmp.executeSynchronous()
if self.isFailure() or self.isBuffer:
return
if self.dataInput.subtractedCurve is not None:
subtracted = self.dataInput.subtractedCurve.path.value
else:
subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
xsdIn = XSDataInputDatop(inputCurve=[
XSDataFile(XSDataString(self.curve)),
XSDataFile(XSDataString(self.hplc_run.first_curve))
],
outputCurve=XSDataFile(
XSDataString(subtracted)),
operation=XSDataString("sub"))
self.__edPluginDatop = self.loadPlugin(self.strControlledPluginDatop)
self.__edPluginDatop.dataInput = xsdIn
self.__edPluginDatop.connectSUCCESS(self.doSuccessDatop)
self.__edPluginDatop.connectFAILURE(self.doFailureDatop)
self.__edPluginDatop.executeSynchronous()
if self.subtracted and os.path.exists(self.subtracted):
xsdIn = XSDataInputSaxsAnalysis(
scatterCurve=XSDataFile(XSDataString(self.subtracted)),
gnomFile=XSDataFile(
XSDataString(
os.path.splitext(self.subtracted)[0] + ".out")))
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = xsdIn
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_0.postProcess")
if self.hplc_run.buffer:
self.xsDataResult.bufferCurve = XSDataFile(
XSDataString(self.hplc_run.buffer))
if self.curve:
self.xsDataResult.integratedCurve = XSDataFile(
XSDataString(self.curve))
if self.subtracted:
self.xsDataResult.subtractedCurve = XSDataFile(
XSDataString(self.subtracted))
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
executiveSummary = os.linesep.join(self.lstExecutiveSummary)
self.xsDataResult.status = XSDataStatus(
executiveSummary=XSDataString(executiveSummary))
self.dataOutput = self.xsDataResult
if self.frame:
self.frame.processing = False
def average_buffers(self):
"""
Average out all buffers
"""
self.lstExecutiveSummary.append("Averaging out buffer files: " +
", ".join(self.hplc_run.for_buffer))
xsdIn = XSDataInputDataver(inputCurve=[
XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer
])
if self.dataInput.bufferCurve:
xsdIn.outputCurve = self.dataInput.bufferCurve
else:
xsdIn.outputCurve = XSDataFile(
XSDataString(
self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] +
"_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer)))
self.__edPluginDatAver = self.loadPlugin(
self.strControlledPluginDatAver)
self.__edPluginDatAver.dataInput = xsdIn
self.__edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
self.__edPluginDatAver.connectFAILURE(self.doFailureDatAver)
self.__edPluginDatAver.executeSynchronous()
def doSuccessProcessOneFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
output = _edPlugin.dataOutput
if not output.integratedCurve:
strErr = "Edna plugin ProcessOneFile did not produce integrated curve"
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
return
self.curve = output.integratedCurve.path.value
if not os.path.exists(self.curve):
strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!"
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
return
self.xsDataResult.integratedCurve = output.integratedCurve
self.xsDataResult.normalizedImage = output.normalizedImage
if output.experimentSetup and output.experimentSetup.timeOfFrame:
startTime = output.experimentSetup.timeOfFrame.value
else:
startTime = None
with self._sem:
if not self.hplc_run.first_curve:
self.hplc_run.first_curve = self.curve
self.hplc_run.start_time = startTime
self.frame.curve = self.curve
self.frame.time = startTime
def doFailureProcessOneFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append(
"Edna plugin ProcessOneFile failed.")
self.setFailure()
def doSuccessDatop(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatop")
self.retrieveSuccessMessages(_edPlugin,
"EDPluginBioSaxsHPLCv1_0.doSuccessDatop")
if _edPlugin and _edPlugin.dataOutput:
output = _edPlugin.dataOutput
if output.status and output.status.executiveSummary:
self.lstExecutiveSummary.append(
output.status.executiveSummary.value)
if output.outputCurve:
self.subtracted = output.outputCurve.path.value
if os.path.exists(self.subtracted):
self.xsDataResult.subtractedCurve = output.outputCurve
self.frame.subtracted = self.subtracted
else:
strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doFailureDatop(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatop")
self.retrieveFailureMessages(_edPlugin,
"EDPluginBioSaxsHPLCv1_0.doFailureDatop")
strErr = "Edna plugin datop failed."
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append(strErr)
self.ERROR(strErr)
self.setFailure()
def doSuccessSaxsAnalysis(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
gnom = _edPlugin.dataOutput.gnom
if gnom:
if gnom.rgGnom:
self.frame.gnom = gnom.rgGnom.value
if gnom.dmax:
self.frame.Dmax = gnom.dmax.value
if gnom.total:
self.frame.total = gnom.total.value
self.xsDataResult.gnom = gnom
volume = _edPlugin.dataOutput.volume
if volume:
self.frame.volume = volume.value
self.xsDataResult.volume = volume
rg = _edPlugin.dataOutput.autoRg
if rg:
if rg.rg:
self.frame.Rg = rg.rg.value
if rg.rgStdev:
self.frame.Rg_Stdev = rg.rgStdev.value
if rg.i0:
self.frame.I0 = rg.i0.value
if rg.i0Stdev:
self.frame.I0_Stdev = rg.i0Stdev.value
if rg.quality:
self.frame.quality = rg.quality.value
self.xsDataResult.autoRg = rg
def doFailureSaxsAnalysis(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis")
strErr = "Edna plugin SaxsAnalysis failed."
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
self.lstExecutiveSummary.append(strErr)
else:
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doSuccessDatCmp(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity:
fidelity = _edPlugin.dataOutput.fidelity.value
else:
strErr = "No fidelity in output of datcmp"
self.error(strErr)
self.lstExecutiveSummary.append(strErr)
#self.setFailure()
fidelity = 0
if self.hplc_run.buffer is None:
if fidelity > 0:
self.isBuffer = True
if fidelity > 0.1:
self.hplc_run.for_buffer.append(self.curve)
else:
self.average_buffers()
elif fidelity > 0:
self.isBuffer = True
def doFailureDatCmp(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatCmp")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatCmp")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append("Edna plugin DatCmp failed.")
self.setFailure()
def doSuccessDatAver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatAver")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatAver")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve:
bufferCurve = _edPlugin.dataOutput.outputCurve.path.value
if os.path.exists(bufferCurve):
with self._sem:
self.hplc_run.buffer = bufferCurve
else:
strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doFailureDatAver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatAver")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatAver")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append("Edna plugin DatAver failed.")
self.setFailure()
class EDObject(object):
"""
Virtual base class for all EDNA Objects (classes).
It offers some synchronization and locking capabilities to make the code thread safe.
"""
__semaphoreId = Semaphore()
__iId_class = 0
def __init__(self):
"""
Constructor of the main pure virtual class.
This constructor implements:
- the creation of the semaphore
- definition of timer object (uninitialized as potentially not used)
"""
object.__init__(self)
with self.__class__.__semaphoreId:
self.__class__.__iId_class += 1
self.__iId = self.__class__.__iId_class
self.__semaphore = Semaphore()
self.__fTimeInit = None
self.__fTimeEnd = None
self.__classname = None
def getId(self):
return self.__iId
def getClassName(self):
"""
Retrieves the name of the class
@return: the name of the class
@rtype: string
"""
return self.__class__.__name__
def synchronizeOn(self):
"""
This method must be used in together with the method synchronizeOff().
This method makes the code threadsafe till the method synchronizeOff
is called.
"""
self.__semaphore.acquire()
def synchronizeOff(self):
"""
This method must be used in together with the method synchronizeOn().
"""
self.__semaphore.release()
def locked(self):
return self.__semaphore
def setTimeInit(self):
"""
Initializes the timer for the object
"""
if self.__fTimeInit is None:
self.__fTimeInit = time.time()
def getTimeInit(self):
"""
Retrieves the time of initialization
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeInit
def setTimeEnd(self):
"""
Set the end of calculation time for the given object
"""
if self.__fTimeEnd is None:
self.__fTimeEnd = time.time()
def getTimeEnd(self):
"""
Retrieves the time of end of task
@return: number of seconds since epoch
@rtype: float
"""
return self.__fTimeEnd
def getRunTime(self):
"""
@returns: the RunTime for the given object
@rtype: float
"""
fRetrunRunTime = 0.0
if self.__fTimeInit is not None:
if self.__fTimeEnd is None:
fRetrunRunTime = time.time() - self.__fTimeInit
else:
fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
return fRetrunRunTime
class EDPluginExecPyFAIv1_0(EDPluginExec):
"""
This is the basic plugin of PyFAI for azimuthal integration
"""
_dictGeo = {} #key:tuple(ai.param), value= ai
_sem = Semaphore()
def __init__(self):
"""
"""
EDPluginExec.__init__(self)
self.setXSDataInputClass(XSDataInputPyFAI)
self.shared = None
self.strOutputFile = None
self.ai = None #this is the azimuthal integrator to use
self.data = None
self.mask = None
self.nbPt = None
self.dummy = None
self.delta_dummy = None
self.npaOut = None
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginExecPyFAIv1_0.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
# self.checkMandatoryParameters(self.dataInput.geometry, "No Geometry given")
# self.checkMandatoryParameters(self.dataInput.detector, "No Detector given")
self.checkMandatoryParameters(self.dataInput.input, "No input image given")
self.checkMandatoryParameters(self.dataInput.nbPt, "No Number of points given for integration")
@staticmethod
def getDetector(xsDetector):
detector = None
if xsDetector.name and (xsDetector.name.value in dir(pyFAI.detectors)):
detector = getattr(pyFAI.detectors, xsDetector.name.value)()
else:
pixel2 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeX)
pixel1 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeY)
if xsDetector.splineFile and os.path.isFile(xsDetector.splineFile.path.value):
dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":xsDetector.splineFile.path.value}
else:
dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":None}
detector = pyFAI.detectors.Detector()
detector.setPyFAI(**dictGeo)
return detector
def preProcess(self, _edObject=None):
EDPluginExec.preProcess(self)
self.DEBUG("EDPluginExecPyFAIv1_0.preProcess")
sdi = self.dataInput
ai = pyFAI.AzimuthalIntegrator()
if sdi.geometryFit2D is not None:
xsGeometry = sdi.geometryFit2D
detector = self.getDetector(xsGeometry.detector)
d = {"direct": EDUtilsUnit.getSIValue(xsGeometry.distance) * 1000, #fit2D takes the distance in mm
"centerX": xsGeometry.beamCentreInPixelsX.value ,
"centerY":xsGeometry.beamCentreInPixelsY.value ,
"tilt": xsGeometry.angleOfTilt.value,
"tiltPlanRotation": xsGeometry.tiltRotation.value}
d.update(detector.getFit2D())
ai.setFit2D(**d)
elif sdi.geometryPyFAI is not None:
xsGeometry = sdi.geometryPyFAI
detector = self.getDetector(xsGeometry.detector)
d = {"dist": EDUtilsUnit.getSIValue(xsGeometry.sampleDetectorDistance),
"poni1": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence1),
"poni2": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence2),
"rot1": EDUtilsUnit.getSIValue(xsGeometry.rotation1),
"rot2": EDUtilsUnit.getSIValue(xsGeometry.rotation2),
"rot3": EDUtilsUnit.getSIValue(xsGeometry.rotation3)}
d.update(detector.getPyFAI())
ai.setPyFAI(**d)
else:
strError = "Geometry definition in %s, not recognized as a valid geometry%s %s" % (sdi, os.linesep, sdi.marshal())
self.ERROR(strError)
raise RuntimeError(strError)
########################################################################
# Choose the azimuthal integrator
########################################################################
with self.__class__._sem:
if tuple(ai.param) in self.__class__._dictGeo:
self.ai = self.__class__._dictGeo[tuple(ai.param)]
else:
self.__class__._dictGeo[tuple(ai.param)] = ai
self.ai = ai
self.data = EDUtilsArray.getArray(self.dataInput.input).astype(float)
if sdi.dark is not None:
self.data -= EDUtilsArray.getArray(sdi.dark)
if sdi.flat is not None:
self.data /= EDUtilsArray.getArray(sdi.flat)
if sdi.mask is not None:
self.mask = EDUtilsArray.getArray(sdi.mask)
if sdi.wavelength is not None:
self.ai.wavelength = EDUtilsUnit.getSIValue(sdi.wavelength)
if sdi.output is not None:
self.strOutputFile = sdi.output.path.value
if sdi.dummy is not None:
self.dummy = sdi.dummy.value
if sdi.deltaDummy is not None:
self.delta_dummy = sdi.deltaDummy.value
if sdi.nbPt:
self.nbPt = sdi.nbPt.value
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecPyFAIv1_0.process")
data = EDUtilsArray.getArray(self.dataInput.input)
if self.dataInput.saxsWaxs and self.dataInput.saxsWaxs.value.lower().startswith("s"):
out = self.ai.saxs(self.data,
nbPt=self.nbPt,
filename=self.strOutputFile,
mask=self.mask,
dummy=self.dummy,
delta_dummy=self.delta_dummy)
else:
out = self.ai.xrpd(self.data,
nbPt=self.nbPt,
filename=self.strOutputFile,
mask=self.mask,
dummy=self.dummy,
delta_dummy=self.delta_dummy)
self.npaOut = np.hstack((i.reshape(-1, 1) for i in out if i is not None))
def postProcess(self, _edObject=None):
EDPluginExec.postProcess(self)
self.DEBUG("EDPluginExecPyFAIv1_0.postProcess")
# Create some output data
if self.strOutputFile:
output = XSDataImageExt(path=XSDataString(self.strOutputFile))
elif self.shared:
output = XSDataImageExt(shared=XSDataString(self.shared))
else:
output = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaOut))
xsDataResult = XSDataResultPyFAI(output=output)
self.setDataOutput(xsDataResult)
class EDPluginWaitFile(EDPlugin):
"""
Plugins that waits for a file to be written and reach a certain size
"""
#offers some extra seconds between officiel timeout and allows the plugin to finish gracefully.
EXTRA_TIME = 5
DELTA_TIME = 1
DEFAULT_TIMEOUT = 2
config_timeout = None
writeXmlInOut = True
writeDataXMLOutput = True
writeDataXMLInput = True
sem = Semaphore()
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.setXSDataInputClass(XSDataInputWaitFile)
self.__filename = None
self.__exists = None
self.__filesize = None
self.__expectedSize = None
self.__timeout = self.DEFAULT_TIMEOUT
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginWaitFile.checkParameters")
self.checkMandatoryParameters(self.getDataInput(),
"Data Input is None")
self.checkMandatoryParameters(self.getDataInput().getExpectedFile(),
"Data Input is None")
self.checkMandatoryParameters(self.getDataInput().getExpectedSize(),
"Data Input is None")
def configure(self):
"""
Configuration step of the plugin: mainly extend the timeout by 5 seconds to let the plugin finish.
"""
if self.__class__.config_timeout is None:
with self.__class__.sem:
if self.__class__.config_timeout is None:
iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None)
if iTimeOut is not None:
self.DEBUG(
"EDPlugin.configure: Setting time out to %d s from plugin configuration."
% iTimeOut)
self.__class__.config_timeout = iTimeOut
else:
self.__class__.config_timeout = self.getDefaultTimeOut(
)
self.__class__.writeXmlInOut = bool(
self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT,
True))
self.__class__.writeDataXMLOutput = bool(
self.config.get(self.CONF_WRITE_XML_OUTPUT, True))
self.__class__.writeDataXMLInput = bool(
self.config.get(self.CONF_WRITE_XML_INPUT, True))
self.__timeout = self.__class__.config_timeout
self.setTimeOut(self.__class__.config_timeout +
EDPluginWaitFile.EXTRA_TIME)
self.setWriteXMLInputOutput(self.writeXmlInOut)
self.setWriteXMLOutput(self.writeDataXMLOutput)
self.setWriteXMLInput(self.writeDataXMLInput)
def preProcess(self, _edObject=None):
EDPlugin.preProcess(self)
self.DEBUG("EDPluginWaitFile.preProcess")
self.__filename = self.getDataInput().getExpectedFile().getPath(
).getValue()
self.__expectedSize = self.getDataInput().getExpectedSize().getValue()
if self.getDataInput().getTimeOut():
self.__timeout = self.getDataInput().getTimeOut().getValue()
self.setTimeOut(self.getDataInput().getTimeOut().getValue() +
EDPluginWaitFile.EXTRA_TIME)
def process(self, _edObject=None):
EDPlugin.process(self)
self.DEBUG("EDPluginWaitFile.process")
self.DEBUG(
"EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s"
% (self.getTimeOut(), self.__timeout))
self.setTimeInit()
dirname = os.path.dirname(self.__filename)
while self.getRunTime() < self.__timeout:
if os.path.isdir(dirname):
fd = os.open(dirname, os.O_RDONLY)
os.fstat(fd)
os.close(fd)
if os.path.exists(self.__filename):
self.__filesize = os.path.getsize(self.__filename)
if self.__filesize >= self.__expectedSize:
break
time.sleep(EDPluginWaitFile.DELTA_TIME)
self.setTimeEnd()
def postProcess(self, _edObject=None):
EDPlugin.postProcess(self)
self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" %
self.getRunTime())
xsDataResult = XSDataResultWaitFile()
if os.path.exists(self.__filename):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__filename))
xsDataResult.setActualFile(xsDataFile)
xsDataResult.setActualSize(
XSDataInteger(os.path.getsize(self.__filename)))
xsDataResult.setTimedOut(
XSDataBoolean(self.getRunTime() >= self.__timeout))
# Create some output data
self.setDataOutput(xsDataResult)
class Reprocess(object):
EDNAPluginName = "EDPluginBioSaxsProcessOneFilev1_4"
hc = 12.398419292004204
def __init__(self):
self.XML = "<XSDataInput>\
<normalizedImage><path><value>${FULLPATH}</value></path></normalizedImage>\
<correctedImage><path><value>${DIRDIRNAME}/2d/${BASENAME}.edf</value></path></correctedImage>\
<normalizedImageSize><value>4100000</value></normalizedImageSize>\
<integratedCurve><path><value>${DIRDIRNAME}/1d/${BASENAME}.edf</value></path></integratedCurve>\
<maskFile><path><value>${MASKFILE}</value></path></maskFile>\
<code><value>BSA</value></code>\
</XSDataInput>"
self.maskfile = None
self.dataFiles = []
self.wavelength = 1.0
self.debug = False
self.mode = "offline"
self.newerOnly = False
self.nbcpu = multiprocessing.cpu_count()
self.cpu_sem = Semaphore(self.nbcpu)
self.process_sem = Semaphore()
self.queue = Queue()
def fileName2xml(self, filename):
"""Here we create the XML string to be passed to the EDNA plugin from the input filename
This can / should be modified by the final user
@param filename: full path of the input file
@type filename: python string representing the path
@rtype: XML string
@return: python string
"""
if filename.endswith(".edf"):
FULLPATH = os.path.abspath(filename)
DIRNAME, NAME = os.path.split(FULLPATH)
DIRDIRNAME = os.path.dirname(DIRNAME)
BASENAME, EXT = os.path.splitext(NAME)
if not os.path.isdir(os.path.join(DIRDIRNAME, "1d")):
os.makedirs(os.path.join(DIRDIRNAME, "1d"), int("775", 8))
return self.xml.replace("${FULLPATH}", FULLPATH).\
replace("${DIRNAME}", DIRNAME).replace("${NAME}", NAME).\
replace("${DIRDIRNAME}", DIRDIRNAME).replace("${BASENAME}", BASENAME).\
replace("${EXT}", EXT).replace("$MASKFILE", self.maskfile or "")
def XMLerr(self, strXMLin):
"""
This is an example of XMLerr function ... it prints only the name of the file created
@param srXMLin: The XML string used to launch the job
@type strXMLin: python string with the input XML
@rtype: None
@return: None
"""
self.cpu_sem.release()
if type(strXMLin) not in types.StringTypes:
strXMLin= strXMLin.marshal()
EDVerbose.WARNING("Error in the processing of :\n%s" % strXMLin)
return None
def XMLsuccess(self, strXMLin):
"""
This is an example of XMLerr function ... it prints only the name of the file created
@param srXMLin: The XML string used to launch the job
@type strXMLin: python string with the input XML
@rtype: None
@return: None
"""
self.cpu_sem.release()
# EDVerbose.WARNING("Error in the processing of :\n%s" % strXMLin)
return None
def parse(self):
"""
parse options from command line
"""
parser = optparse.OptionParser()
parser.add_option("-V", "--version", dest="version", action="store_true",
help="print version of the program and quit", metavar="FILE", default=False)
parser.add_option("-v", "--verbose",
action="store_true", dest="debug", default=False,
help="switch to debug/verbose mode")
parser.add_option("-m", "--mask", dest="mask",
help="file containing the mask (for image reconstruction)", default=None)
parser.add_option("-M", "--mode", dest="mode",
help="Mode can be online/offline/all", default="offline")
parser.add_option("-o", "--out", dest="output",
help="file for log", default=None)
parser.add_option("-w", "--wavelength", dest="wavelength", type="float",
help="wavelength of the X-Ray beam in Angstrom", default=None)
parser.add_option("-e", "--energy", dest="energy", type="float",
help="energy of the X-Ray beam in keV (hc=%skeV.A)" % self.hc, default=None)
parser.add_option("-t", "--template", dest="template", type="str",
help="template XML file", default=None)
parser.add_option("-n", "--nbcpu", dest="nbcpu", type="int",
help="template XML file", default=self.nbcpu)
(options, args) = parser.parse_args()
# Analyse aruments and options
if options.version:
print("BioSaxs Azimuthal integration version %s" % __version__)
sys.exit(0)
if options.debug:
EDVerbose.setVerboseDebugOn()
self.debug = True
if options.output:
EDVerbose.setLogFileName(options.output)
if options.mask and os.path.isfile(options.mask):
self.maskfile = options.mask
if options.template and os.path.isfile(options.template):
self.xml = open(options.template).read()
if options.wavelength:
self.wavelength = 1e-10 * options.wavelength
elif options.energy:
self.wavelength = 1e-10 * self.hc / options.energy
if options.mode=="offline":
self.mode = "offline"
self.newerOnly = False
elif options.mode=="online":
self.mode = "dirwarch"
self.newerOnly = True
elif options.mode=="dirwatch":
self.mode = "dirwarch"
self.newerOnly = False
self.cpu_sem = Semaphore(options.nbcpu)
self.nbcpu = options.nbcpu
self.dataFiles = [f for f in args if os.path.isfile(f)]
if not self.dataFiles:
raise RuntimeError("Please provide datafiles or read the --help")
def process(self):
for fn in self.dataFiles:
EDVerbose.screen("Processing file %s" % fn)
edj = EDJob(self.EDNAPluginName)
edj.dataInput = self.fileName2xml(fn)
edj.connectSUCCESS(self.XMLsuccess)
edj.connectFAILURE(self.XMLerr)
self.queue.put(edj)
if self.process_sem._Semaphore__value > 0 :
t = threading.Thread(target=self.startProcessing)
t.start()
EDVerbose.screen("Back in main")
while self.cpu_sem._Semaphore__value < self.nbcpu:
time.sleep(0.1)
EDJob.synchronizeAll()
EDJob.stats()
def startProcessing(self):
with self.process_sem:
while not self.queue.empty():
self.cpu_sem.acquire()
edj = self.queue.get()
edj.execute()
def __init__(self, runId, first_curve=None):
self.id = runId
self.buffer = None
self.first_curve = first_curve
self.frames = {} #key: id, value: HPLCframe instance
self.curves = []
self.for_buffer = []
self.hdf5_filename = None
self.hdf5 = None
self.chunk_size = 250
self.lock = Semaphore()
if first_curve:
self.files.append(first_curve)
self.max_size = None
self.start_time = None
self.time = None
self.gnom = None
self.Dmax = None
self.total = None
self.volume = None
self.Rg = None
self.Rg_Stdev = None
self.I0 = None
self.I0_Stdev = None
self.quality = None
self.q = None
self.size = None
self.scattering_I = None
self.scattering_Stdev = None
self.subtracted_I = None
self.subtracted_Stdev = None
self.sum_I = None
self.Vc = None
self.Qr = None
self.mass = None
self.Vc_Stdev = None
self.Qr_Stdev = None
self.mass_Stdev = None
self.buffer_frames = None
self.merge_frames = None
self.buffer_I = None
self.buffer_Stdev = None
self.merge_I = None
self.merge_Stdev = None
self.merge_curves = []
self.merge_Rg = {}
self.merge_analysis = {}
self.merge_framesDIC = {}
self.keys1d = [
"gnom", "Dmax", "total", "volume", "Rg", "Rg_Stdev", "I0",
"I0_Stdev", "quality", "sum_I", "Vc", "Qr", "mass", "Vc_Stdev",
"Qr_Stdev", "mass_Stdev"
]
self.keys2d = [
"scattering_I", "scattering_Stdev", "subtracted_I",
"subtracted_Stdev"
]
self.keys_frames = ["buffer_frames", "merge_frames"]
self.keys_merges = [
"buffer_I", "buffer_Stdev", "merge_I", "merge_Stdev"
]
class EDPluginExecMeasureOffsetv1_0(EDPluginExec):
"""
An exec plugin that taked two images and measures the offset between the two.
"""
__sem = Semaphore()
CONF_CONVOLUTION = None
CONF_CONVOLUTION_KEY = "convolution"
CONF_CONVOLUTION_DEFAULT = "numpy" #can be "fftw", "cuda" or simply falls back on numpy.fftpack
def __init__(self):
"""
"""
EDPluginExec.__init__(self)
self.setXSDataInputClass(XSDataInputMeasureOffset)
self.npaIm1 = None
self.npaIm2 = None
self.tOffset = None
self.tCrop = None
self.tCenter = None
self.tWidth = None
self.tSmooth = None
self.bBackgroundsubtraction = False
self.sobel = False
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginExecMeasureOffsetv1_0.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
def configure(self):
"""
Configures the plugin from the configuration file with the following parameters
- The CONVOLUTION method
"""
EDPluginExec.configure(self)
if self.CONF_CONVOLUTION is None:
with self.__class__.__sem:
self.DEBUG("EDPluginExecMeasureOffsetv1_0.configure")
xsPluginItem = self.getConfiguration()
if xsPluginItem is None:
self.WARNING(
"EDPluginExecMeasureOffsetv1_0.configure: No plugin item defined."
)
xsPluginItem = XSPluginItem()
strCONVOLUTION = EDConfiguration.getStringParamValue(
xsPluginItem, self.CONF_CONVOLUTION_KEY)
if strCONVOLUTION is None:
self.WARNING("EDPluginExecMeasureOffsetv1_0.configure: No configuration parameter found for: %s using default value: %s\n%s"\
% (self.CONF_CONVOLUTION_KEY, self.CONF_CONVOLUTION_DEFAULT, xsPluginItem.marshal()))
self.__class__CONF_CONVOLUTION = self.CONF_CONVOLUTION_DEFAULT
else:
self.__class__.CONF_CONVOLUTION = strCONVOLUTION.strip(
).lower()
def preProcess(self, _edObject=None):
EDPluginExec.preProcess(self)
self.DEBUG("EDPluginExecMeasureOffsetv1_0.preProcess")
sdi = self.dataInput
images = sdi.image
arrays = sdi.array
if len(images) == 2:
self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0]))
self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1]))
elif len(arrays) == 2:
self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0])
self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1])
else:
strError = "EDPluginExecMeasureOffsetv1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal(
)[:1000]
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
crop = sdi.cropBorders
if len(crop) > 1:
self.tCrop = tuple([i.value for i in crop])
elif len(crop) == 1:
self.tCrop = (crop[0].value, crop[0].value)
center = sdi.center
if len(center) > 1:
self.tCenter = tuple([i.value for i in center])
elif len(center) == 1:
self.tCenter = (center[0].value, center[0].value)
width = sdi.width
if len(width) > 1:
self.tWidth = tuple([i.value for i in width])
elif len(width) == 1:
self.tWidth = (width[0].value, width[0].value)
smooth = sdi.smoothBorders
if len(smooth) == 2:
self.tSmooth = (smooth[0].value, smooth[1].value)
elif len(smooth) == 1:
self.tSmooth = (smooth[0].value, smooth[0].value)
if sdi.backgroundSubtraction is not None:
self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value
in [1, True, "true"])
if sdi.sobelFilter is not None:
self.sobel = (sdi.sobelFilter in [1, True, "true"])
EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape,
"Images have the same size")
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecMeasureOffsetv1_0.process")
shape = self.npaIm1.shape
if (self.tCrop is not None):
d0min = min(shape[0], self.tCrop[0])
d1min = min(shape[1], self.tCrop[1])
d0max = max(0, shape[0] - self.tCrop[0])
d1max = max(0, shape[1] - self.tCrop[1])
shape = (d0max - d0min, d1max - d1min)
else:
d0min = 0
d0max = shape[0]
d1min = 0
d1max = shape[1]
if self.tCenter is None:
#the default center is the geometry center of the image ...
self.tCenter = [i // 2 for i in shape]
if self.tWidth is not None:
d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2))
d0max = min(shape[0], d0min + self.tWidth[0])
d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2))
d1max = min(shape[1], d1min + self.tWidth[1])
shape = (d0max - d0min, d1max - d1min)
if shape != self.npaIm1.shape:
self.DEBUG(
"Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s"
% (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter,
self.tWidth))
self.npaIm1 = self.npaIm1[d0min:d0max, d1min:d1max]
self.npaIm2 = self.npaIm2[d0min:d0max, d1min:d1max]
shape = self.npaIm1.shape
self.DEBUG("After Crop, images have shape : %s and %s " %
(self.npaIm1.shape, self.npaIm2.shape))
if self.bBackgroundsubtraction:
self.DEBUG("performing background subtraction")
x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range(
shape[0] - 1, 0, -1) + [0] * (shape[0] - 1)
y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * (
shape[1] - 1) + range(shape[1] - 1, 0, -1)
spline1 = scipy.interpolate.SmoothBivariateSpline(
x,
y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))],
kx=1,
ky=1)
spline2 = scipy.interpolate.SmoothBivariateSpline(
x,
y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))],
kx=1,
ky=1)
self.npaIm1 -= spline1(range(shape[0]), range(shape[1]))
self.npaIm2 -= spline2(range(shape[0]), range(shape[1]))
if self.tSmooth is not None:
self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth)
self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth)
if self.sobel:
self.npaIm1 = scipy.ndimage.sobel(self.npaIm1)
self.npaIm2 = scipy.ndimage.sobel(self.npaIm2)
self.DEBUG("Doing ffts using %s" % self.CONF_CONVOLUTION)
offset, logs = MeasureOffset.measure_offset(self.npaIm1,
self.npaIm2,
self.CONF_CONVOLUTION,
withLog=True)
self.tOffset = [XSDataDouble(f) for f in offset]
logs.insert(0, "")
if len(logs) <= 4:
self.DEBUG(os.linesep.join(logs))
else:
self.WARNING(os.linesep.join(logs))
def postProcess(self, _edObject=None):
EDPluginExec.postProcess(self)
self.DEBUG("EDPluginExecMeasureOffsetv1_0.postProcess")
# Create some output data
self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
from __future__ import with_statement
__author__ = "Jérôme Kieffer"
__license__ = "GPLv3+"
__copyright__ = "2011 ESRF"
__status__ = "production"
__date__ = "20120917"
import os, shutil
from EDPluginControl import EDPluginControl
from XSDataCommon import XSDataFile, XSDataString, XSDataStatus
from XSDataEdnaSaxs import XSDataInputAutoSub, XSDataInputDataver, \
XSDataInputDatcmp, XSDataInputAutoRg, XSDataInputDatop
from XSDataEdnaSaxs import XSDataResultAutoSub
from EDThreading import Semaphore
_sem = Semaphore()
def copy(src, dst):
"""
Operating system independent copy that does symbolic links on posix system.
This implements a lock to be extremely thread-safe as BsxCube call many times the same plugin ... at the same time
"""
with _sem:
if os.path.exists(src):
if os.name == "posix":
if os.path.islink(dst):
os.unlink(dst)
os.symlink(src, dst)
else:
class EDPluginExecNormalizeImagev1_1(EDPluginExec):
"""
Plugin that normalizes an image (subtract dark current and divide by flat-field image, taking into account the exposure time)
v1.1 is compatible with EDShare -> lower memory footprint
"""
dictDark = {} #Nota: the key is a string: str(exposutTime)
# listDarkExposure = []
# listDarkArray = []
semaphore = Semaphore()
dtype = None #
CONF_DTYPE_KEY = "dtype"
CONF_DTYPE_DEFAULT = "float32"
def __init__(self):
"""
Constructor of the plugin
"""
EDPluginExec.__init__(self)
self.setXSDataInputClass(XSDataInputNormalize)
self.listDataArray = []
self.listDataExposure = []
self.listDarkArray = []
self.listDarkExposure = []
self.listFlatArray = []
self.listFlatExposure = []
self.npaNormalized = None
self.strOutputFilename = None
self.strOutputShared = None
self.shape = None
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginExecNormalizeImagev1_1.checkParameters")
self.checkMandatoryParameters(self.getDataInput(), "Data Input is None")
def configure(self):
"""
Add the dtype to the configuration file, probably "float32" or "float64"
"""
EDPluginExec.configure(self)
self.DEBUG("EDPluginExecNormalizeImagev1_1.configure")
if self.__class__.dtype is None:
with self.__class__.semaphore:
xsPluginItem = self.getConfiguration()
if (xsPluginItem == None):
self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No plugin item defined.")
xsPluginItem = XSPluginItem()
strDtype = EDConfiguration.getStringParamValue(xsPluginItem, self.CONF_DTYPE_KEY)
if(strDtype == None):
self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No configuration parameter found for: %s using default value: %s\n%s"\
% (self.CONF_DTYPE_KEY, self.CONF_DTYPE_DEFAULT, xsPluginItem.marshal()))
self.__class__.dtype = self.CONF_DTYPE_DEFAULT
else:
self.__class__.dtype = str(strDtype.strip().lower())
def preProcess(self, _edObject=None):
EDPluginExec.preProcess(self)
self.DEBUG("EDPluginExecNormalizeImagev1_1.preProcess")
sdi = self.getDataInput()
if sdi.getData() == []:
strError = "You should either provide at least ONE input filename or an array, you provided: %s" % sdi.marshal()
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
else:
for inputData in sdi.getData():
if inputData.exposureTime is None:
self.WARNING("You did not provide an exposure time for DATA... using default: 1")
self.listDataExposure.append(1.0)
else:
self.listDataExposure.append(EDUtilsUnit.getSIValue(inputData.exposureTime))
if inputData.path is not None:
strPath = inputData.path.value
if os.path.isfile(strPath):
self.listDataArray.append(fabio.open(strPath).data)
else:
strError = "The input file provided for DATA is not a valid file: %s" % strPath
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
elif inputData.array is not None:
self.listDataArray.append(EDUtilsArray.xsDataToArray(inputData.array))
else:
strError = "You should either provide an input filename or an array for DATA, you provided: %s" % inputData.marshal()
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
for inputFlat in sdi.getFlat():
if inputFlat.exposureTime is None:
self.WARNING("You did not provide an exposure time for FLAT... using default: 1")
expTime = 1.0
else:
expTime = EDUtilsUnit.getSIValue(inputFlat.exposureTime)
self.listFlatExposure.append(expTime)
if inputFlat.path is not None:
strPath = inputFlat.path.value
if os.path.isfile(strPath):
self.listFlatArray.append(fabio.open(strPath).data)
else:
strError = "The input file provided for FLAT is not a valid file: %s" % strPath
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
elif inputFlat.array is not None:
self.listFlatArray.append(EDUtilsArray.xsDataToArray(inputFlat.array))
else:
strError = "You should either provide an input filename or an array for FLAT, you provided: %s" % inputFlat.marshal()
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
with EDPluginExecNormalizeImagev1_1.semaphore:
for inputDark in sdi.getDark():
if inputDark.exposureTime is None:
self.WARNING("You did not provide an exposure time for Dark... using default: 1")
expTime = 1.0
else:
expTime = EDUtilsUnit.getSIValue(inputDark.exposureTime)
strMeanDarkKey = "/".join((self.getClassName(), "MeanDark%6.3f" % expTime))
if strMeanDarkKey not in EDShare:
self.listDarkExposure.append(expTime)
if inputDark.path is not None:
strPath = inputDark.path.value
if os.path.isfile(strPath):
self.listDarkArray.append(fabio.open(strPath).data)
else:
strError = "The input file provided for Dark is not a valid file: %s" % strPath
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
elif inputDark.array is not None:
self.listDarkArray.append(EDUtilsArray.xsDataToArray(inputDark.array))
elif inputDark.shared is not None:
self.listDarkArray.append(EDShare[inputDark.shared.value])
else:
strError = "You should either provide an input filename or an array for Dark, you provided: %s" % inputDark.marshal()
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
if sdi.output is not None:
if (sdi.output.path is not None):
self.strOutputFilename = sdi.output.path.value
elif (sdi.output.shared is not None):
self.strOutputShared = sdi.output.shared.value
# else export as array.
EDAssert.equal(len(self.listDataArray), len(self.listDataExposure), _strComment="number of data images / exposure times ")
EDAssert.equal(len(self.listFlatArray), len(self.listFlatExposure), _strComment="number of flat images / exposure times ")
EDAssert.equal(len(self.listDarkArray), len(self.listDarkExposure), _strComment="number of dark images / exposure times ")
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecNormalizeImagev1_1.process")
#numerator part:
fTotalDataTime = 0.0
self.shape = self.listDataArray[0].shape
npaSummedData = numpy.zeros(self.shape, dtype=self.dtype)
for i in range(len(self.listDataArray)):
fTotalDataTime += self.listDataExposure[i]
npaSummedData += numpy.maximum(self.listDataArray[i] - self.getMeanDark(self.listDataExposure[i]), 0)
npaNormalizedData = (npaSummedData / fTotalDataTime).astype(self.dtype)
#denominator part
fTotalFlatTime = 0.0
npaSummedFlat = numpy.zeros(self.shape, dtype=self.dtype)
for i in range(len(self.listFlatArray)):
fTotalFlatTime += self.listFlatExposure[i]
npaSummedFlat += numpy.maximum(self.listFlatArray[i] - self.getMeanDark(self.listFlatExposure[i]), 0)
npaNormalizedFlat = (npaSummedFlat / fTotalFlatTime).astype(self.dtype)
self.npaNormalized = npaNormalizedData / numpy.maximum(npaNormalizedFlat, numpy.ones_like(npaNormalizedFlat))
if self.npaNormalized.dtype != numpy.dtype(self.dtype):
self.npaNormalized = self.npaNormalized.astype(self.dtype)
def postProcess(self, _edObject=None):
EDPluginExec.postProcess(self)
self.DEBUG("EDPluginExecNormalizeImagev1_1.postProcess")
xsDataResult = XSDataResultNormalize()
if self.strOutputFilename is not None:
self.DEBUG("Writing file %s" % self.strOutputFilename)
edf = fabio.edfimage.edfimage(data=self.npaNormalized, header={})
edf.write(self.strOutputFilename)
xsdo = XSDataImageExt(path=XSDataString(self.strOutputFilename))
elif self.strOutputShared is not None:
self.DEBUG("EDShare --> %s" % self.strOutputShared)
EDShare[ self.strOutputShared] = self.npaNormalized
xsdo = XSDataImageExt(shared=XSDataString(self.strOutputShared))
else:
xsdo = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaNormalized))
xsDataResult.output = xsdo
# Create some output data
self.setDataOutput(xsDataResult)
def finallyProcess(self, _edObject=None):
"""
after processing of the plugin:
remove reference to large objects to save memory
"""
self.DEBUG("EDPluginExecNormalizeImagev1_1.finallyProcess")
EDPluginExec.finallyProcess(self, _edObject)
self.listDataArray = None
self.listDataExposure = None
self.listFlatArray = None
self.listFlatExposure = None
self.npaNormalized = None
def getMeanDark(self, _fExposureTime):
"""
@param _fExposureTime: exposure time
@return: mean of darks with this exposure time
"""
with self.__class__.semaphore:
if str(_fExposureTime) not in self.__class__.dictDark:
npaSumDark = numpy.zeros(self.shape, dtype="float64")
count = 0
for fExpTime, npaDark in zip(self.listDarkExposure, self.listDarkArray):
if abs(fExpTime - _fExposureTime) / _fExposureTime < 1e-4:
npaSumDark += npaDark
count += 1
if count == 0:
self.WARNING("No Dark image with Exposure time = %.3f, no dark subtraction" % _fExposureTime)
self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark
else:
self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark / float(count)
return self.__class__.dictDark[str(_fExposureTime)]
class EDPluginBioSaxsHPLCv1_2(EDPluginControl):
"""
plugin for processing Saxs data coming from HPLC
runs subsequently:
*ProcessOneFile,
*subtraction of buffer
*AutoRg
todo:
only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs
Changes since v1.0:
* do not perfom Gnom not Porod analysis
* return Summed Intensity and time stamp
* fix issue with timestamp.
Changes since v1.1:
* Read configuration for a buffer threshold
"""
strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_4"
strControlledPluginDatop = "EDPluginExecDatopv2_0"
strControlledPluginAutoRg = "EDPluginExecAutoRgv1_0"
strControlledPluginDatCmp = "EDPluginExecDatcmpv2_0"
strControlledPluginDatAver = "EDPluginExecDataverv2_0"
dictHPLC = {} # key=runId, value= HPLCrun instance
_sem = Semaphore()
SIMILARITY_THRESHOLD_SAMPLE_KEY = "similaritySample"
SIMILARITY_THRESHOLD_BUFFER_KEY = "similarityBuffer"
SIMILARITY_THRESHOLD_SAMPLE_DEFAULT = 0.0
SIMILARITY_THRESHOLD_BUFFER_DEFAULT = 0.1
SIMILARITY_THRESHOLD_SAMPLE = None
SIMILARITY_THRESHOLD_BUFFER = None
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0)
self.edPluginProcessOneFile = None
self.edPluginSubtract = None
self.edPluginAutoRg = None
self.edPluginDatCmp = None
self.xsDataResult = XSDataResultBioSaxsHPLCv1_0()
self.runId = None
self.frameId = None
self.frame = None
self.hplc_run = None
self.curve = None
self.subtracted = None
self.lstExecutiveSummary = []
self.isBuffer = False
self.scatter_data = None
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginBioSaxsHPLCv1_2.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image")
self.checkMandatoryParameters(self.dataInput.sample,
"no Sample parameter")
self.checkMandatoryParameters(self.dataInput.experimentSetup,
"No experimental setup parameter")
def configure(self):
"""
Configures the HPLC plugin by reading from the configuration file
- The threshold for similarity with sample & buffer
"""
EDPluginControl.configure(self)
if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None:
with self._sem:
if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None:
self.DEBUG("EDPluginBioSaxsHPLCv1_2.configure")
self.__class__.SIMILARITY_THRESHOLD_BUFFER = float(
self.config.get(
self.SIMILARITY_THRESHOLD_BUFFER_KEY,
self.SIMILARITY_THRESHOLD_BUFFER_DEFAULT))
self.__class__.SIMILARITY_THRESHOLD_SAMPLE = float(
self.config.get(
self.SIMILARITY_THRESHOLD_SAMPLE_KEY,
self.SIMILARITY_THRESHOLD_SAMPLE_DEFAULT))
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_2.preProcess")
sdi = self.dataInput
if sdi.runId is not None:
self.runId = sdi.runId.value
else:
path = sdi.rawImage.path.value
if "_" in path:
self.runId = path[::-1].split("_", 1)[1][::-1]
else:
self.runId = path
with self._sem:
if self.runId not in self.dictHPLC:
self.dictHPLC[self.runId] = HPLCrun(self.runId)
self.hplc_run = self.dictHPLC[self.runId]
if sdi.frameId is not None:
self.frameId = sdi.frameId.value
else:
path = sdi.rawImage.path.value
if "_" in path:
digits = os.path.splitext(
os.path.basename(path))[0].split("_")[-1]
try:
self.frameId = int(digits)
except ValueError:
self.WARNING(
"frameId is supposed to be an integer, I got %s" %
digits)
self.frameId = digits
else:
self.warning("using frameID=0 in tests, only")
self.frameId = 0
with self._sem:
self.frame = HPLCframe(self.runId, self.frameId)
self.hplc_run.frames[self.frameId] = self.frame
if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value):
with self._sem:
self.hplc_run.buffer = sdi.bufferCurve.path.value
if self.hplc_run.hdf5_filename:
hplc = self.hplc_run.hdf5_filename
elif sdi.hplcFile:
hplc = sdi.hplcFile.path.value
else:
path = sdi.rawImage.path.value
dirname = os.path.dirname(os.path.dirname(path))
hplc_dir = os.path.join(dirname, "HPLC")
if not os.path.exists(hplc_dir):
try:
os.mkdir(hplc_dir)
except:
pass # I don't care
if "_" in path:
hplc = "_".join(
os.path.splitext(
os.path.basename(path))[0].split("_")[:-1]) + ".h5"
else:
hplc = os.path.splitext(os.path.basename(path))[0] + ".h5"
hplc = os.path.join(hplc_dir, hplc)
if not self.hplc_run.hdf5_filename:
with self._sem:
self.hplc_run.init_hdf5(hplc)
# self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc))
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_2.process")
xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(
rawImage=self.dataInput.rawImage,
sample=self.dataInput.sample,
experimentSetup=self.dataInput.experimentSetup,
rawImageSize=self.dataInput.rawImageSize,
normalizedImage=self.dataInput.normalizedImage,
integratedCurve=self.dataInput.integratedCurve,
runId=self.dataInput.runId,
frameId=self.dataInput.frameId)
self.edPluginProcessOneFile = self.loadPlugin(
self.strControlledPluginProcessOneFile)
self.edPluginProcessOneFile.dataInput = xsdIn
self.edPluginProcessOneFile.connectSUCCESS(
self.doSuccessProcessOneFile)
self.edPluginProcessOneFile.connectFAILURE(
self.doFailureProcessOneFile)
self.edPluginProcessOneFile.executeSynchronous()
if self.isFailure():
return
xsdIn = XSDataInputDatcmp(inputCurve=[
XSDataFile(XSDataString(self.hplc_run.first_curve)),
XSDataFile(XSDataString(self.curve))
])
self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
self.edPluginDatCmp.dataInput = xsdIn
self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
self.edPluginDatCmp.executeSynchronous()
if self.isFailure() or self.isBuffer:
return
if self.dataInput.subtractedCurve is not None:
subtracted = self.dataInput.subtractedCurve.path.value
else:
subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
if self.hplc_run.buffer is not None:
xsdIn = XSDataInputDatop(inputCurve=[
XSDataFile(XSDataString(self.curve)),
XSDataFile(XSDataString(self.hplc_run.buffer))
],
outputCurve=XSDataFile(
XSDataString(subtracted)),
operation=XSDataString("sub"))
else:
xsdIn = XSDataInputDatop(inputCurve=[
XSDataFile(XSDataString(self.curve)),
XSDataFile(XSDataString(self.hplc_run.first_curve))
],
outputCurve=XSDataFile(
XSDataString(subtracted)),
operation=XSDataString("sub"))
self.edPluginDatop = self.loadPlugin(self.strControlledPluginDatop)
self.edPluginDatop.dataInput = xsdIn
self.edPluginDatop.connectSUCCESS(self.doSuccessDatop)
self.edPluginDatop.connectFAILURE(self.doFailureDatop)
self.edPluginDatop.executeSynchronous()
if self.subtracted and os.path.exists(self.subtracted):
self.edPluginAutoRg = self.loadPlugin(
self.strControlledPluginAutoRg)
self.edPluginAutoRg.dataInput = XSDataInputAutoRg(
inputCurve=[XSDataFile(XSDataString(self.subtracted))])
self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg)
self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg)
self.edPluginAutoRg.executeSynchronous()
def postProcess(self, _edObject=None):
"""
after process
"""
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_2.postProcess")
if self.hplc_run.buffer:
self.xsDataResult.bufferCurve = XSDataFile(
XSDataString(self.hplc_run.buffer))
if self.curve:
self.xsDataResult.integratedCurve = XSDataFile(
XSDataString(self.curve))
if self.subtracted:
self.xsDataResult.subtractedCurve = XSDataFile(
XSDataString(self.subtracted))
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
executiveSummary = os.linesep.join(self.lstExecutiveSummary)
self.xsDataResult.status = XSDataStatus(
executiveSummary=XSDataString(executiveSummary))
self.dataOutput = self.xsDataResult
if self.frame:
self.frame.processing = False
def average_buffers(self):
"""
Average out all buffers
"""
self.lstExecutiveSummary.append("Averaging out buffer files: " +
", ".join(self.hplc_run.for_buffer))
xsdIn = XSDataInputDataver(inputCurve=[
XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer
])
if self.dataInput.bufferCurve:
xsdIn.outputCurve = self.dataInput.bufferCurve
else:
xsdIn.outputCurve = XSDataFile(
XSDataString(
self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] +
"_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer)))
self.edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver)
self.edPluginDatAver.dataInput = xsdIn
self.edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
self.edPluginDatAver.connectFAILURE(self.doFailureDatAver)
self.edPluginDatAver.executeSynchronous()
def doSuccessProcessOneFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
output = _edPlugin.dataOutput
if not output.integratedCurve:
strErr = "Edna plugin ProcessOneFile did not produce integrated curve"
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
return
self.curve = output.integratedCurve.path.value
if not os.path.exists(self.curve):
strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!"
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
return
self.xsDataResult.integratedCurve = output.integratedCurve
self.xsDataResult.normalizedImage = output.normalizedImage
self.xsDataResult.dataI = output.dataI
self.xsDataResult.dataQ = output.dataQ
self.xsDataResult.dataStdErr = output.dataStdErr
if output.experimentSetup and output.experimentSetup.timeOfFrame:
startTime = output.experimentSetup.timeOfFrame.value
else:
try:
startTime = float(
fabio.openheader(self.dataInput.rawImage.path.value).
header["time_of_day"])
except Exception:
self.ERROR("Unable to read time_of_day in header of %s" %
self.dataInput.rawImage.path.value)
startTime = 0
with self._sem:
if not self.hplc_run.first_curve:
self.hplc_run.first_curve = self.curve
self.hplc_run.start_time = startTime
self.frame.curve = self.curve
self.frame.time = startTime
self.xsDataResult.timeStamp = XSDataTime(
value=(startTime - self.hplc_run.start_time))
self.calcIntensity()
def doFailureProcessOneFile(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append(
"Edna plugin ProcessOneFile failed.")
self.setFailure()
def doSuccessDatop(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatop")
self.retrieveSuccessMessages(_edPlugin,
"EDPluginBioSaxsHPLCv1_2.doSuccessDatop")
if _edPlugin and _edPlugin.dataOutput:
output = _edPlugin.dataOutput
if output.status and output.status.executiveSummary:
self.lstExecutiveSummary.append(
output.status.executiveSummary.value)
if output.outputCurve:
self.subtracted = output.outputCurve.path.value
if os.path.exists(self.subtracted):
self.xsDataResult.subtractedCurve = output.outputCurve
self.frame.subtracted = self.subtracted
else:
strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doFailureDatop(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatop")
self.retrieveFailureMessages(_edPlugin,
"EDPluginBioSaxsHPLCv1_2.doFailureDatop")
strErr = "Edna plugin datop failed."
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append(strErr)
self.ERROR(strErr)
self.setFailure()
def doSuccessAutoRg(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
try:
rg = _edPlugin.dataOutput.autoRgOut[0]
except:
self.ERROR("AutoRg returned nothing !")
else:
if rg.rg:
self.frame.Rg = rg.rg.value
if rg.rgStdev:
self.frame.Rg_Stdev = rg.rgStdev.value
if rg.i0:
self.frame.I0 = rg.i0.value
if rg.i0Stdev:
self.frame.I0_Stdev = rg.i0Stdev.value
if rg.quality:
self.frame.quality = rg.quality.value
self.xsDataResult.autoRg = rg
"""
Calculate the invariants Vc and Qr from the Rambo&Tainer 2013 Paper,
also the the mass estimate based on Qr for proteins
"""
if self.subtracted and os.path.exists(self.subtracted):
self.subtracted_data = numpy.loadtxt(self.subtracted)
if self.subtracted_data is not None and\
self.frame.Rg and self.frame.Rg_Stdev and self.frame.I0 and self.frame.I0_Stdev:
dictRTI = RamboTainerInvariant(self.subtracted_data,
self.frame.Rg,
self.frame.Rg_Stdev,
self.frame.I0,
self.frame.I0_Stdev,
rg.firstPointUsed.value)
# {'Vc': vc[0], 'dVc': vc[1], 'Qr': qr, 'dQr': dqr, 'mass': mass, 'dmass': dmass}
self.frame.Vc = dictRTI.get("Vc")
self.frame.Vc_Stdev = dictRTI.get("dVc")
self.frame.Qr = dictRTI.get("Qr")
self.frame.Qr_Stdev = dictRTI.get("dQ")
self.frame.mass = dictRTI.get("mass")
self.frame.mass_Stdev = dictRTI.get("dmass")
xsdRTI = XSDataRamboTainer(
vc=XSDataDouble(self.frame.Vc),
qr=XSDataDouble(self.frame.Qr),
mass=XSDataDouble(self.frame.mass),
dvc=XSDataDouble(self.frame.Vc_Stdev),
dqr=XSDataDouble(self.frame.Qr_Stdev),
dmass=XSDataDouble(self.frame.mass_Stdev))
self.xsDataResult.rti = xsdRTI
def doFailureAutoRg(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureAutoRg")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureAutoRg")
strErr = "Edna plugin AutoRg failed."
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
self.lstExecutiveSummary.append(strErr)
else:
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doSuccessDatCmp(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity:
fidelity = _edPlugin.dataOutput.fidelity.value
else:
strErr = "No fidelity in output of datcmp"
self.error(strErr)
self.lstExecutiveSummary.append(strErr)
# self.setFailure()
fidelity = 0
if self.hplc_run.buffer is None:
if fidelity > self.SIMILARITY_THRESHOLD_SAMPLE:
self.isBuffer = True
if fidelity > self.SIMILARITY_THRESHOLD_BUFFER:
self.hplc_run.for_buffer.append(self.curve)
else:
self.average_buffers()
elif fidelity > self.SIMILARITY_THRESHOLD_SAMPLE:
self.isBuffer = True
def doFailureDatCmp(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatCmp")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatCmp")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append("Edna plugin DatCmp failed.")
self.setFailure()
def doSuccessDatAver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatAver")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatAver")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve:
bufferCurve = _edPlugin.dataOutput.outputCurve.path.value
if os.path.exists(bufferCurve):
with self._sem:
self.hplc_run.buffer = bufferCurve
else:
strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve
self.ERROR(strErr)
self.lstExecutiveSummary.append(strErr)
self.setFailure()
def doFailureDatAver(self, _edPlugin=None):
self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatAver")
self.retrieveFailureMessages(
_edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatAver")
if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
self.lstExecutiveSummary.append(
_edPlugin.dataOutput.status.executiveSummary.value)
else:
self.lstExecutiveSummary.append("Edna plugin DatAver failed.")
self.setFailure()
def calcIntensity(self):
"""
Calculate invarients like:
Sum(I),
and set output datastructure.
"""
self.scatter_data = numpy.loadtxt(
self.xsDataResult.integratedCurve.path.value)
self.frame.sum_I = self.scatter_data[:, 1].sum()
self.xsDataResult.summedIntensity = XSDataDouble(
value=self.frame.sum_I)
class EDPluginExecAlignFramev1_0(EDPluginExec):
"""
An exec plugin that taked an images and a reference and alines the frame on the reference based on the SIFT algorithm.
Derived from EDPluginExecMeasureOffsetv2_1.py
"""
use_sift_pyocl = None
sift_keypoints = None
sift_match = None
keyindex = {
} #key = md5 of the image; value = keypoint (as numpy recordarray)
config_lock = Semaphore()
def __init__(self):
"""
"""
EDPluginExec.__init__(self)
self.setXSDataInputClass(XSDataInputMeasureOffset)
self.npaIm1 = None
self.npaIm2 = None
self.tOffset = None
self.tCrop = None
self.tCenter = None
self.tWidth = None
self.tSmooth = None
self.bBackgroundsubtraction = False
self.sobel = False
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginExecAlignFramev1_0.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
def configure(self):
"""
Configures the plugin (if not done yet
"""
EDPluginExec.configure(self)
if self.use_sift_pyocl is None:
with self.config_lock:
if self.use_sift_pyocl is None:
self.DEBUG("EDPluginExecAlignFramev1_0.configure")
self.__class__.use_sift_pyocl = self.config.get(
"sift_pyocl", False)
print self.use_sift_pyocl, self.sift_keypoints, self.sift_match
def calc_plan(self, image=None):
if self.sift_keypoints is None:
with self.config_lock:
if self.sift_keypoints is None:
if self.use_sift_pyocl and sift_pyocl:
if self.config.get("device", None):
self.__class__.sift_match = sift_pyocl.MatchPlan(
device=self.config["device"]).match
self.__class__.sift_keypoints = sift_pyocl.SiftPlan(
template=image,
device=self.config["device"]).keypoints
else:
deviceType = self.config.get("deviceType", "cpu")
self.__class__.sift_match = sift_pyocl.MatchPlan(
devicetype=deviceType).match
self.__class__.sift_keypoints = sift_pyocl.SiftPlan(
template=image,
devicetype=deviceType).keypoints
elif feature:
self.__class__.sift_match = feature.sift_match
self.__class__.sift_keypoints = feature.sift_keypoints
else:
strerr = "Impossible to make plan: use_sift_pyocl=%s but feature=%s and sift_pyocl=%s" % (
self.use_sift_pyoc, feature, sift_pyocl)
self.ERROR(strerr)
raise RuntimeError(strerr)
print self.use_sift_pyocl, self.sift_keypoints, self.sift_match
def preProcess(self, _edObject=None):
EDPluginExec.preProcess(self)
self.DEBUG("EDPluginExecAlignFramev1_0.preProcess")
sdi = self.dataInput
images = sdi.image
arrays = sdi.array
if len(images) == 2:
self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0]))
self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1]))
elif len(arrays) == 2:
self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0])
self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1])
else:
strError = "EDPluginExecAlignFramev1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal(
)[:1000]
self.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
crop = sdi.cropBorders
if len(crop) > 1:
self.tCrop = tuple([i.value for i in crop])
elif len(crop) == 1:
self.tCrop = (crop[0].value, crop[0].value)
center = sdi.center
if len(center) > 1:
self.tCenter = tuple([i.value for i in center])
elif len(center) == 1:
self.tCenter = (center[0].value, center[0].value)
width = sdi.width
if len(width) > 1:
self.tWidth = tuple([i.value for i in width])
elif len(width) == 1:
self.tWidth = (width[0].value, width[0].value)
smooth = sdi.smoothBorders
if len(smooth) == 2:
self.tSmooth = (smooth[0].value, smooth[1].value)
elif len(smooth) == 1:
self.tSmooth = (smooth[0].value, smooth[0].value)
if sdi.backgroundSubtraction is not None:
self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value
in [1, True, "true"])
if sdi.sobelFilter is not None:
self.sobel = (sdi.sobelFilter in [1, True, "true"])
EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape,
"Images have the same size")
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecAlignFramev1_0.process")
shape = self.npaIm1.shape
if (self.tCrop is not None):
d0min = min(shape[0], self.tCrop[0])
d1min = min(shape[1], self.tCrop[1])
d0max = max(0, shape[0] - self.tCrop[0])
d1max = max(0, shape[1] - self.tCrop[1])
shape = (d0max - d0min, d1max - d1min)
else:
d0min = 0
d0max = shape[0]
d1min = 0
d1max = shape[1]
if self.tCenter is None:
#the default center is the geometry center of the image ...
self.tCenter = [i // 2 for i in shape]
if self.tWidth is not None:
d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2))
d0max = min(shape[0], d0min + self.tWidth[0])
d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2))
d1max = min(shape[1], d1min + self.tWidth[1])
shape = (d0max - d0min, d1max - d1min)
if shape != self.npaIm1.shape:
self.DEBUG(
"Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s"
% (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter,
self.tWidth))
#array contiguity is needed for checksum calculation
self.npaIm1 = numpy.ascontiguousarray(self.npaIm1[d0min:d0max,
d1min:d1max])
self.npaIm2 = numpy.ascontiguousarray(self.npaIm2[d0min:d0max,
d1min:d1max])
shape = self.npaIm1.shape
self.DEBUG("After Crop, images have shape : %s and %s " %
(self.npaIm1.shape, self.npaIm2.shape))
if self.bBackgroundsubtraction:
self.DEBUG("performing background subtraction")
x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range(
shape[0] - 1, 0, -1) + [0] * (shape[0] - 1)
y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * (
shape[1] - 1) + range(shape[1] - 1, 0, -1)
spline1 = scipy.interpolate.SmoothBivariateSpline(
x,
y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))],
kx=1,
ky=1)
spline2 = scipy.interpolate.SmoothBivariateSpline(
x,
y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))],
kx=1,
ky=1)
self.npaIm1 -= spline1(range(shape[0]), range(shape[1]))
self.npaIm2 -= spline2(range(shape[0]), range(shape[1]))
if self.tSmooth is not None:
self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth)
self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth)
if self.sobel:
self.npaIm1 = scipy.ndimage.sobel(self.npaIm1)
self.npaIm2 = scipy.ndimage.sobel(self.npaIm2)
self.calc_plan(self.npaIm1)
checksum1 = hashlib.md5(self.npaIm1).hexdigest()
checksum2 = hashlib.md5(self.npaIm2).hexdigest()
if checksum1 not in self.keyindex:
self.keyindex[checksum1] = self.sift_keypoints(self.npaIm1)
if checksum2 not in self.keyindex:
self.keyindex[checksum2] = self.sift_keypoints(self.npaIm2)
data = self.sift_match(self.keyindex[checksum1],
self.keyindex[checksum2])
v0 = data[:, 0].y - data[:, 1].y
v1 = data[:, 0].x - data[:, 1].x
self.tOffset = [
XSDataDouble(numpy.median(v0)),
XSDataDouble(numpy.median(v1))
]
def postProcess(self, _edObject=None):
EDPluginExec.postProcess(self)
self.DEBUG("EDPluginExecAlignFramev1_0.postProcess")
# Create some output data
self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
class EDPluginControlAlignStackv1_0(EDPluginControl):
"""
This control plugin aligns a stack on images in a single HDF5 3D-structure
"""
__iRefFrame = None
__dictRelShift = {
} #key=frame number N, value= 2-tuple of shift relative to frame N-1
__dictAbsShift = {
} #key=frame number N, value= 2-tuple of shift relative to frame iRefFrame
__semaphore = Semaphore()
MaxOffset = None
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputAlignStack)
self.iFrames = []
self.npArrays = []
self.xsdMeasureOffset = None
self.hdf5ExtraAttributes = None
self.xsdHDF5File = None
self.xsdHDF5Internal = None
self.bAlwaysMOvsRef = True
self.bDoAlign = True
self.semAccumulator = Semaphore()
self.semMeasure = Semaphore()
self.semShift = Semaphore()
self.lstSem = [
self.locked(), self.semAccumulator, self.semMeasure, self.semShift
]
self.queue = Queue()
self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0"
self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0"
self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0"
self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1"
self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10"
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginControlAlignStackv1_0.checkParameters")
self.checkMandatoryParameters(self.dataInput, "Data Input is None")
self.checkMandatoryParameters(self.dataInput.getHDF5File(),
"No Output HDF5 file provided")
self.checkMandatoryParameters(self.dataInput.getInternalHDF5Path(),
"No output HDF5 internal path provided")
self.checkMandatoryParameters(self.dataInput.images,
"No images to process provided")
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAlignStackv1_0.preProcess")
sdi = self.dataInput
self.xsdHDF5File = sdi.HDF5File
self.xsdHDF5Internal = sdi.internalHDF5Path
self.hdf5ExtraAttributes = sdi.extraAttributes
if sdi.dontAlign is not None:
self.bDoAlign = not (bool(sdi.dontAlign.value))
self.iFrames = [xsd.value for xsd in sdi.index]
for idx, oneXSDFile in enumerate(sdi.images):
self.npArrays.append(EDUtilsArray.getArray(oneXSDFile))
if len(self.iFrames) <= idx:
if (oneXSDFile.number is not None):
self.iFrames.append(oneXSDFile.number.value)
elif oneXSDFile.path is not None:
number = ""
filename = oneXSDFile.path.value
basename = os.path.splitext(filename)[0]
for i in basename[-1:0:-1]:
if i.isdigit():
number = i + number
else:
break
self.iFrames.append(int(number))
if self.npArrays == []:
strError = "EDPluginControlAlignStackv1_0.preProcess: You should either provide an images or an arrays, but I got: %s" % sdi.marshal(
)
self.ERROR(strError)
self.setFailure()
self.xsdMeasureOffset = sdi.measureOffset
if self.xsdMeasureOffset.alwaysVersusRef is not None:
self.bAlwaysMOvsRef = bool(
self.xsdMeasureOffset.alwaysVersusRef.value)
with self.__class__.__semaphore:
if (self.__class__.__iRefFrame is None):
self.DEBUG("reference Frame is: %s" % sdi.frameReference.value)
if sdi.getFrameReference() is not None:
self.__class__.__iRefFrame = sdi.frameReference.value
else:
self.__class__.__iRefFrame = 0
if len(self.iFrames) == len(self.npArrays):
for i, j in zip(self.iFrames, self.npArrays):
self.addFrame(i, j)
else:
self.ERROR(
"EDPluginControlAlignStackv1_0.preProcess: You should either provide an images with a frame number or precise it in the XML ! I got: %s"
% sdi.marshal())
self.setFailure()
raise RuntimeError
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlAlignStackv1_0.process")
for iFrame in self.iFrames:
edPluginExecAccumulator = self.loadPlugin(
self.__strControlledPluginAccumulator)
queryRaw = XSDataQuery()
queryShift = XSDataQuery()
queryRaw.setRemoveItems(XSDataBoolean(False))
queryShift.setRemoveItems(XSDataBoolean(False))
xsdataAcc = XSDataInputAccumulator()
if (EDPluginControlAlignStackv1_0.__iRefFrame
== iFrame) or (self.bDoAlign == False):
EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame] = (0.0,
0.0)
EDPluginControlAlignStackv1_0.__dictRelShift[iFrame] = (0.0,
0.0)
self.hdf5_offset(index=iFrame, offset=[0.0, 0.0])
edPluginExecShift = self.loadPlugin(
self.__strControlledPluginShift)
xsdata = XSDataInputShiftImage(
index=XSDataInteger(iFrame),
offset=[
XSDataDouble(i) for i in
EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame]
],
inputImage=self.getFrameRef(iFrame),
outputImage=XSDataImageExt(
shared=XSDataString("Shifted-%06i" % iFrame)))
edPluginExecShift.setDataInput(xsdata)
edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage)
edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage)
self.queue.put(edPluginExecShift)
if (self.bDoAlign == False):
self.executeControlledPlugins()
return
elif EDPluginControlAlignStackv1_0.__iRefFrame < iFrame:
if self.bAlwaysMOvsRef:
queryRaw.setItem([
XSDataString(
"raw %04i" %
(EDPluginControlAlignStackv1_0.__iRefFrame)),
XSDataString("raw %04i" % iFrame)
])
xsdataAcc.setQuery([queryRaw])
else:
queryRaw.setItem([
XSDataString("raw %04i" % (iFrame - 1)),
XSDataString("raw %04i" % iFrame)
])
queryShift.setItem([
XSDataString("shift %04i" % i) for i in range(
EDPluginControlAlignStackv1_0.__iRefFrame +
1, iFrame + 1)
])
xsdataAcc.setQuery([queryRaw, queryShift])
elif EDPluginControlAlignStackv1_0.__iRefFrame > iFrame:
if self.bAlwaysMOvsRef:
queryRaw.setItem([
XSDataString("raw %04i" % iFrame),
XSDataString(
"raw %04i" %
(EDPluginControlAlignStackv1_0.__iRefFrame))
])
xsdataAcc.setQuery([queryRaw])
else:
queryRaw.setItem([
XSDataString("raw %04i" % (iFrame + 1)),
XSDataString("raw %04i" % iFrame)
])
queryShift.setItem([
XSDataString("shift %04i" % i) for i in range(
EDPluginControlAlignStackv1_0.__iRefFrame -
1, iFrame - 1, -1)
])
xsdataAcc.setQuery([queryRaw, queryShift])
if (EDPluginControlAlignStackv1_0.__iRefFrame == iFrame):
self.saveReferenceFrame(iFrame)
xsdataAcc.setItem([XSDataString("raw %04i" % iFrame)])
edPluginExecAccumulator.setDataInput(xsdataAcc)
edPluginExecAccumulator.connectSUCCESS(
self.doSuccessExecAccumultor)
edPluginExecAccumulator.connectFAILURE(
self.doFailureExecAccumulator)
self.queue.put(edPluginExecAccumulator)
self.executeControlledPlugins()
def executeControlledPlugins(self):
"""
Execute all plugins under control:
"""
bAllFinished = False
while not bAllFinished:
#acquire all semaphores to be sure no plugins are under configuration !
for sem in self.lstSem:
with sem:
pass
if self.queue.empty():
self.synchronizePlugins()
bAllFinished = self.queue.empty()
else:
while not self.queue.empty():
try:
plugin = self.queue.get()
except Exception:
self.WARNING(
"In EDPluginControlAlignStackv1_0, exception in self.queue.get()"
)
break
else:
#this is a hack to prevent thousands of threads to be launched at once.
with EDUtilsParallel.getSemaphoreNbThreads():
pass
plugin.execute()
self.synchronizePlugins()
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlAlignStackv1_0.postProcess")
self.executeControlledPlugins()
self.synchronizePlugins()
# Create some output data
xsDataResult = XSDataResultAlignStack()
xsDataResult.setHDF5File(self.xsdHDF5File)
xsDataResult.setInternalHDF5Path(self.xsdHDF5Internal)
self.setDataOutput(xsDataResult)
self.emptyListOfLoadedPlugin()
def doSuccessExecMeasureOffset(self, _edPlugin=None):
with self.semMeasure:
self.DEBUG(
"EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset")
self.retrieveSuccessMessages(
_edPlugin,
"EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset")
listIndex = [i.getValue() for i in _edPlugin.dataInput.index]
listIndex.sort()
dataOutput = _edPlugin.dataOutput
if self.bAlwaysMOvsRef:
if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame:
iToShift, iRef = tuple(listIndex)
EDPluginControlAlignStackv1_0.__dictAbsShift[
iToShift] = tuple(
[-i.getValue() for i in dataOutput.getOffset()])
else:
iRef, iToShift = tuple(listIndex)
EDPluginControlAlignStackv1_0.__dictAbsShift[
iToShift] = tuple(
[i.getValue() for i in dataOutput.getOffset()])
self.screen(
"Frame number %i has absolute offset of %.3f,%.3f" %
(iToShift,
EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][0],
EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][1])
)
edPluginExecShift = self.loadPlugin(
self.__strControlledPluginShift)
xsdata = XSDataInputShiftImage(
index=XSDataInteger(iToShift),
offset=[
XSDataDouble(i) for i in
EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift]
],
inputImage=self.getFrameRef(iToShift),
outputImage=XSDataImageExt(
shared=XSDataString("Shifted-%06i" % iToShift)))
edPluginExecShift.setDataInput(xsdata)
edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage)
edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage)
self.queue.put(edPluginExecShift)
else:
if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame:
iToShift, iRef = tuple(listIndex)
EDPluginControlAlignStackv1_0.__dictRelShift[
iToShift] = tuple(
[-i.value for i in dataOutput.offset])
else:
iRef, iToShift = tuple(listIndex)
EDPluginControlAlignStackv1_0.__dictRelShift[
iToShift] = tuple([i.value for i in dataOutput.offset])
self.screen(
"Frame number %i has relative offset of %.3f,%.3f" %
(iToShift,
EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][0],
EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][1])
)
xsdata = XSDataInputAccumulator(
item=[XSDataString("shift %04i" % iToShift)])
edPluginExecAccumulator = self.loadPlugin(
self.__strControlledPluginAccumulator)
edPluginExecAccumulator.setDataInput(xsdata)
edPluginExecAccumulator.connectSUCCESS(
self.doSuccessExecAccumultor)
edPluginExecAccumulator.connectFAILURE(
self.doFailureExecAccumulator)
self.queue.put(edPluginExecAccumulator)
def doFailureExecMeasureOffset(self, _edPlugin=None):
self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset")
self.retrieveFailureMessages(
_edPlugin,
"EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset")
self.ERROR(
"Failure in execution of the MeasureOffset with input: %s and output %s"
% (_edPlugin.dataInput.marshal()[:1000],
_edPlugin.dataOutput.marshal()[:1000]))
self.setFailure()
def doSuccessExecShiftImage(self, _edPlugin=None):
with self.semShift:
edPluginExecHDF5 = self.loadPlugin(self.__strControlledPluginHDF5)
self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecShiftImage")
self.retrieveSuccessMessages(
_edPlugin,
"EDPluginControlAlignStackv1_0.doSuccessExecShiftImage")
xsdIdx = _edPlugin.dataInput.index
self.__class__.MaxOffset = _edPlugin.MAX_OFFSET_VALUE
self.hdf5_offset(
index=xsdIdx.value,
offset=[i.value for i in _edPlugin.dataInput.offset])
xsdata = XSDataInputHDF5StackImages(
chunkSegmentation=XSDataInteger(32),
forceDtype=XSDataString("float32"),
extraAttributes=self.hdf5ExtraAttributes,
internalHDF5Path=self.xsdHDF5Internal,
HDF5File=self.xsdHDF5File,
index=[xsdIdx],
inputImageFile=[_edPlugin.dataOutput.outputImage])
# inputArray=[_edPlugin.dataOutput.outputArray])
edPluginExecHDF5.setDataInput(xsdata)
edPluginExecHDF5.connectSUCCESS(self.doSuccessExecStackHDF5)
edPluginExecHDF5.connectFAILURE(self.doFailureExecStackHDF5)
self.queue.put(edPluginExecHDF5)
def doFailureExecShiftImage(self, _edPlugin=None):
self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecShiftImage")
self.retrieveFailureMessages(
_edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecShiftImage")
self.ERROR(
"Failure in execution of the ExecShiftImage with input: %s and output %s"
% (_edPlugin.dataInput.marshal()[:1000],
_edPlugin.dataOutput.marshal()[:1000]))
self.setFailure()
def doSuccessExecStackHDF5(self, _edPlugin=None):
self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5")
self.retrieveSuccessMessages(
_edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5")
def doFailureExecStackHDF5(self, _edPlugin=None):
self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecStackHDF5")
self.retrieveFailureMessages(
_edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecStackHDF5")
self.ERROR(
"Failure in execution of the ExecStackHDF5 with input: %s " %
(_edPlugin.dataInput.marshal()[:1000]))
if _edPlugin.dataOutput is not None:
self.ERROR(
"Failure in execution of the ExecStackHDF5 with output %s" %
(_edPlugin.dataOutput.marshal()[:1000]))
def doSuccessExecAccumultor(self, _edPlugin=None):
with self.semAccumulator:
self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecAccumultor")
self.retrieveSuccessMessages(
_edPlugin,
"EDPluginControlAlignStackv1_0.doSuccessExecAccumultor")
for query in _edPlugin.dataOutput.getQuery():
self.addExtraTime(60)
_edPlugin.addExtraTime(60)
accType = query.getItem()[0].getValue().split()[0]
listInt = [
int(i.getValue().split()[1]) for i in query.getItem()
]
if accType == "raw":
listFrame = [self.getFrameRef(i) for i in listInt]
xsdata = XSDataInputMeasureOffset(image=listFrame)
doSIFT = False
if self.xsdMeasureOffset is not None:
xsdata.setCropBorders(
self.xsdMeasureOffset.getCropBorders())
xsdata.setSmoothBorders(
self.xsdMeasureOffset.getSmoothBorders())
xsdata.setBackgroundSubtraction(
self.xsdMeasureOffset.getRemoveBackground())
if self.xsdMeasureOffset.useSift is not None:
doSIFT = self.xsdMeasureOffset.useSift.value
if max(listInt
) > EDPluginControlAlignStackv1_0.__iRefFrame:
listInt.sort()
else:
listInt.sort(reverse=True)
xsdata.setIndex([XSDataInteger(i) for i in listInt])
if doSIFT:
edPluginExecMeasure = self.loadPlugin(
self.__strControlledPluginMeasureSIFT)
else:
edPluginExecMeasure = self.loadPlugin(
self.__strControlledPluginMeasureFFT)
edPluginExecMeasure.setDataInput(xsdata)
edPluginExecMeasure.connectSUCCESS(
self.doSuccessExecMeasureOffset)
edPluginExecMeasure.connectFAILURE(
self.doFailureExecMeasureOffset)
self.queue.put(edPluginExecMeasure)
elif accType == "shift":
shift_1 = 0.0
shift_2 = 0.0
for frame in listInt:
shift_1 += EDPluginControlAlignStackv1_0.__dictRelShift[
frame][0]
shift_2 += EDPluginControlAlignStackv1_0.__dictRelShift[
frame][1]
if listInt[0] > EDPluginControlAlignStackv1_0.__iRefFrame:
iFrameShift = max(listInt)
else:
iFrameShift = min(listInt)
EDPluginControlAlignStackv1_0.__dictAbsShift[
iFrameShift] = (shift_1, shift_2)
self.screen(
"Frame number %i has absolute offset of %.3f,%.3f" %
(iFrameShift, shift_1, shift_2))
edPluginExecShift = self.loadPlugin(
self.__strControlledPluginShift)
edPluginExecShift.dataInput = XSDataInputShiftImage(
index=XSDataInteger(iFrameShift),
offset=[XSDataDouble(shift_1),
XSDataDouble(shift_2)],
inputImage=self.getFrameRef(iFrameShift),
outputImage=XSDataImageExt(
shared=XSDataString("Shifted-%06i" % iFrameShift)))
edPluginExecShift.connectSUCCESS(
self.doSuccessExecShiftImage)
edPluginExecShift.connectFAILURE(
self.doFailureExecShiftImage)
self.queue.put(edPluginExecShift)
self.DEBUG("Items: %s" % EDPluginAccumulatorv1_0.getItems())
self.DEBUG("Queries: %s" % EDPluginAccumulatorv1_0.getQueries())
def doFailureExecAccumulator(self, _edPlugin=None):
self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecAccumulator")
self.retrieveFailureMessages(
_edPlugin,
"EDPluginControlAlignStackv1_0.doFailureExecAccumulator")
self.ERROR(
"Failure in execution of the accumulator with input: %s and output %s"
% (_edPlugin.dataInput.marshal()[:1000],
_edPlugin.dataOutput.marshal()[:1000]))
self.setFailure()
def saveReferenceFrame(self, iFrame):
"""
Save the reference frame to the HDF5 file
@param iFrame: frame number to save as reference frame
"""
ref = "reference_frame"
hdf5file = self.xsdHDF5File.path.value
edpluginHDF5 = self.loadPlugin(self.__strControlledPluginHDF5)
with edpluginHDF5.getFileLock(hdf5file):
#Seems strange to redefine h5Grp but if there is a flush in between: h5Grp could be closed
entry = edpluginHDF5.getHDF5File(hdf5file)[
self.xsdHDF5Internal.value]
if ref in entry:
del entry[ref]
entry[ref] = self.getFrame(iFrame)
entry[ref].attrs["index"] = iFrame
def hdf5_offset(self, index, offset):
with EDPluginHDF5.getFileLock(self.xsdHDF5File.path.value):
grp = EDPluginHDF5.getHDF5File(
self.xsdHDF5File.path.value)[self.xsdHDF5Internal.value]
ds = grp["Offsets"]
if self.MaxOffset:
if "MaxOffset" not in ds.attrs:
ds.attrs["MaxOffset"] = self.MaxOffset
ds[index, :] = offset
@classmethod
def showData(cls):
EDVerbose.screen("*" * 20 + "EDPluginControlAlignStackv1_0" + "*" * 20)
EDVerbose.screen("Reference Frame: %s" % cls.__iRefFrame)
if len(cls.__dictRelShift) < len(cls.__dictAbsShift):
mydict = cls.__dictAbsShift.copy()
else:
mydict = cls.__dictRelShift.copy()
lstTxt = []
for i in mydict:
txt = "Frame %4i:\t" % i
rela = cls.__dictRelShift.get(i)
abso = cls.__dictAbsShift.get(i)
if rela:
txt += "relative: (%.3f, %.3f)\t" % rela
else:
txt += "relative: %12s\t" % rela
if abso:
txt += "absolute: (%.3f, %.3f)" % abso
else:
txt += "absolute: %12s" % abso
lstTxt.append(txt)
EDVerbose.screen(os.linesep.join(lstTxt))
items = EDPluginAccumulatorv1_0.getItems()
items.sort()
EDVerbose.screen("Items in the accumulator: %s" % (items))
querylist = [" "] + [
str(i) for i in EDPluginAccumulatorv1_0.getQueries().keys()
]
EDVerbose.screen("Queries in the accumulator: " +
os.linesep.join(querylist))
@classmethod
def addFrame(cls, index, value):
"""
Just store the value to EDShare
"""
key = "Normalized-%06i" % int(index)
if key not in EDShare:
EDShare[key] = value
@classmethod
def getFrame(cls, index):
"""
Just Retrieves the value from EDShare
"""
return EDShare["Normalized-%06i" % int(index)]
@classmethod
def getFrameRef(cls, index):
"""
Just retrieves the reference in the EDShare store
@return: reference to the frame in EDShare
@rtype: XSDataImageExt
"""
return XSDataImageExt(shared=XSDataString("Normalized-%06i" %
int(index)))
@classmethod
def cleanUp(cls):
"""
Clean up of the dictionary containing images: Left for compatibility reasons
"""
EDShare.flush()
class EDUtilsParallel(object):
"""
A class helping to control the workload on a multi-threaded application.
"""
_iActualNbCPU = None
_semaphore = Semaphore()
_semaphoreInit = Semaphore()
_semaphoreNbThreads = None
_iNbThreads = None
@classmethod
def _detectNumberOfCPUs(cls):
"""
detect the actual number of CPUs and stores it in a class variable
"""
with cls._semaphore:
if cls._iActualNbCPU is not None:
return
iNbCPU = None
#The best way: using python 2.6 or later
try:
import multiprocessing
iNbCPU = multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
iNbCPU = None
# POSIX
if iNbCPU is None:
try:
iNbCPU = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if iNbCPU <= 0:
iNbCPU = None
except (AttributeError, ValueError):
iNbCPU = None
# Windows
if iNbCPU is None:
try:
iNbCPU = int(os.environ['NUMBER_OF_PROCESSORS'])
if iNbCPU <= 0:
iNbCPU = None
except (KeyError, ValueError):
iNbCPU = None
# jython
if iNbCPU is None:
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
iNbCPU = runtime.availableProcessors()
if iNbCPU <= 0:
iNbCPU = None
except ImportError:
iNbCPU = None
# BSD
if iNbCPU is None:
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
iNbCPU = int(scStdout)
if iNbCPU <= 0:
iNbCPU = None
except (OSError, ValueError):
iNbCPU = None
# Linux
if iNbCPU is None:
try:
iNbCPU = open('/proc/cpuinfo').read().count('processor\t:')
if iNbCPU <= 0:
iNbCPU = None
except IOError:
iNbCPU = None
# Solaris
if iNbCPU is None:
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
iNbCPU = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
iNbCPU += 1
if iNbCPU <= 0:
iNbCPU = None
except OSError:
iNbCPU = None
# Other UNIXes (heuristic)
if iNbCPU is None:
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'],
stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
iNbCPU = 0
while '\ncpu' + str(iNbCPU) + ':' in dmesg:
iNbCPU += 1
if iNbCPU <= 0:
iNbCPU = None
except OSError:
iNbCPU = None
#if nothing else works ...
if iNbCPU is None:
iNbCPU = 1
cls._iActualNbCPU = iNbCPU
@classmethod
def detectNumberOfCPUs(cls, _iMaxCPU=sys.maxint, _bForce=False):
"""
class method :
Detects the number of CPUs on a system. Cribbed from pp.
found : http://www.boduch.ca/2009/06/python-cpus.html
@param _iMaxCPU: the maximum number of CPUs allowed
@type _iMaxCPU: integer
@param _bForce: force the number of CPUs for debugging
@type _bForce: boolean
@return: the number of CPUs
@rtype: integer
"""
if not isinstance(_iMaxCPU, int):
_iMaxCPU = sys.maxint
else:
_iMaxCPU = max(1, _iMaxCPU)
if isinstance(_bForce, bool):
if _bForce:
return _iMaxCPU
if cls._iActualNbCPU is None:
cls._detectNumberOfCPUs()
iNbCPU = cls._iActualNbCPU
if iNbCPU < 1:
return 1
elif iNbCPU > _iMaxCPU:
return _iMaxCPU
else:
return iNbCPU
@classmethod
def initializeNbThread(cls, _iNbThread=None):
"""
Class method:
Initialises a semaphore with the right number of threads
@param _iNbThread: the maximum number of threads concurrently running and CPU intensive
@type _iNbThread: integer
"""
with cls._semaphoreInit:
if cls._semaphoreNbThreads is None:
if _iNbThread is None:
_iNbThread = cls.detectNumberOfCPUs()
EDVerbose.DEBUG(
"Initializing EDUtilsParallel semaphoreNbThreads to %s" %
_iNbThread)
cls._iNbThreads = _iNbThread
cls._semaphoreNbThreads = Semaphore(_iNbThread)
else:
if cls._iNbThreads != _iNbThread:
EDVerbose.WARNING(
"cls._semaphoreNbThreads was already initialized to %s, not changing to %s"
% (cls._iNbThreads, _iNbThread))
@classmethod
def uninitializeNbThread(cls):
"""
For testing purpose: un-initialize the semaphore controlling the number of execPlugin running at once
"""
if cls._semaphoreNbThreads is not None:
EDVerbose.DEBUG(
"resetting the semaphore concerning the number of threads.")
cls._semaphoreNbThreads = None
@classmethod
def getSemaphoreValue(cls):
"""
Class method:
getter for the current value of the semaphore counting the CPU-active threads
@return: value of the semaphore (or None if not initialized)
@rtype: int
"""
if cls._semaphoreNbThreads is not None:
return cls._semaphoreNbThreads._Semaphore__value
else:
return None
@classmethod
def getNbRunning(cls):
"""
Class method:
getter for the number of CPU-active threads running
@return: the number of CPU-active threads runnings
@rtype: integer
"""
if cls._semaphoreNbThreads is not None:
return cls._iNbThreads - cls._semaphoreNbThreads._Semaphore__value
else:
return None
@classmethod
def semaphoreNbThreadsAcquire(cls):
"""
Class method:
Method to acquire the semaphore that controls the number of threads running concurrently
"""
if cls._semaphoreNbThreads is not None:
cls._semaphoreNbThreads.acquire()
else:
EDVerbose.DEBUG(
"EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)."
)
@classmethod
def semaphoreNbThreadsRelease(cls):
"""
Class method:
Method to release the semaphore that controls the number of threads running concurrently
"""
if cls._semaphoreNbThreads is not None:
if cls._semaphoreNbThreads._Semaphore__value < cls._iNbThreads:
cls._semaphoreNbThreads.release()
else:
EDVerbose.DEBUG(
"EDUtilsParallel: Unable to release an uninitialized semaphore (NbCPU)."
)
@classmethod
def getSemaphoreNbThreads(cls):
"""
classmethod that returns the semaphore nb threads (to be used with the with_statement)
@return: semaphore controling CPUs
"""
if cls._semaphoreNbThreads is not None:
return cls._semaphoreNbThreads
else:
EDVerbose.DEBUG(
"EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)."
)