def chooseWorker(self):
"""
selects the worker and sets self.worker
"""
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
# self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
EDPluginSPDCorrectv10.__lockWorker.acquire()
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus()))
if (oneWorker.getConfig() == self._SPDconfig) and (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
if self.worker is None:
if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
self.worker = SPDworker()
EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
self.worker.setExecutable(self.getExecutable())
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID))
self.worker.initialize(self._SPDconfig)
self.worker.setTimeOut(self.getTimeOut())
while self.worker is None:
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
bConfigOK = False
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
#bug1 those string must specifically be the same ... change line 248 in spdCake1.5
if (oneWorker.getConfig() == self._SPDconfig):
bConfigOK = True
if (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
self.workerID = idx
if bConfigOK == False:
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
if (oneWorker.getStatus() in ["free", "uninitialized"]):
#Bug2 why doest this work ???
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
oneWorker.initialize(self._SPDconfig)
self.worker = oneWorker
self.workerID = idx
EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker))
time.sleep(0.1)
if self.workerID is None:
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
EDPluginSPDCorrectv10.__lockWorker.release()
def correctMask(self):
"""
Apply spline and tilt correction to the mask if provided in fit2d format.
"""
self.DEBUG("EDPluginSPDCakev1_5.correctMask")
EDPluginSPDCakev1_5.__semaphore.acquire()
t0 = time.time()
# self.screen("%s: preCalc Mask file is in %s" % (self.getId(), self.dictGeometry["MaskFile"]))
if (self.dictGeometry["MaskFile"]
not in EDPluginSPDCakev1_5.__dictMask):
strUnCorrectedEdfMask = self.dictGeometry["MaskFile"]
unCorrectedMask = fabio.open(strUnCorrectedEdfMask)
basename = os.path.join(
self.getSPDCommonDirectory(),
os.path.splitext(os.path.basename(strUnCorrectedEdfMask))[0])
if self.bCorrectTiltMask:
if not isinstance(unCorrectedMask, fabio.edfimage.edfimage):
strUnCorrectedEdfMask = os.path.join(
self.getSPDCommonDirectory(),
os.path.splitext(
os.path.basename(self.dictGeometry["MaskFile"]))[0]
+ ".edf")
unCorrectedEdfMask = fabio.edfimage.edfimage(
header={}, data=unCorrectedMask.data.astype("float32"))
unCorrectedEdfMask.write(strUnCorrectedEdfMask)
imgConfig = self.getSPDConfig()
mskConfig = " ".join([i for i in imgConfig.split() \
if not (i.startswith("mask_file=") or i.startswith("azim_")\
or i.startswith("flood_file=") or i.startswith("dark_file="))] + ["do_dark=0", "dummy=1"])
worker = SPDworker()
worker.setExecutable(self.getExecutable())
worker.setLogFilename(
os.path.join(self.getSPDCommonDirectory(),
"worker-mask.log"))
worker.initialize(mskConfig)
worker.setTimeOut(self.getTimeOut())
dictLog = worker.process(
"outdir=%s/ %s" %
(self.getSPDCommonDirectory(), strUnCorrectedEdfMask))
for oneLog in dictLog:
self.DEBUG("EDPluginSPDCakev1_5.correctMask %s: %s" %
(oneLog, dictLog[oneLog]))
worker.kill(gentle=True)
worker = None
npaCorrected = -(fabio.open(basename + ".cor").data +
0.5).astype("int8")
else:
npaCorrected = -(unCorrectedMask.data).astype("int8")
strCorrectedMask = basename + ".msk"
fabio.edfimage.edfimage(data=npaCorrected, header={
"Dummy": "-1"
}).write(strCorrectedMask)
EDPluginSPDCakev1_5.__dictMask[
self.dictGeometry["MaskFile"]] = strCorrectedMask
EDPluginSPDCakev1_5.__semaphore.release()
newConfig = []
for key in self.getSPDConfig().split():
if key.startswith("mask_file="):
newConfig.append(
'mask_file=%s' %
os.path.abspath(EDPluginSPDCakev1_5.__dictMask[
self.dictGeometry["MaskFile"]]))
else:
newConfig.append(key)
self.setSPDConfig(" ".join(newConfig))
# self.worker.setConfig(self.getSPDConfig())
self.DEBUG(
"%s updating mask %s took %.3fs " %
(self.getId(), self.dictGeometry["MaskFile"], time.time() - t0))
def correctMask(self):
"""
Apply spline and tilt correction to the mask if provided in fit2d format.
"""
self.DEBUG("EDPluginSPDCakev1_5.correctMask")
EDPluginSPDCakev1_5.__semaphore.acquire()
t0 = time.time()
# self.screen("%s: preCalc Mask file is in %s" % (self.getId(), self.dictGeometry["MaskFile"]))
if (self.dictGeometry["MaskFile"] not in EDPluginSPDCakev1_5.__dictMask):
strUnCorrectedEdfMask = self.dictGeometry["MaskFile"]
unCorrectedMask = fabio.open(strUnCorrectedEdfMask)
basename = os.path.join(self.getSPDCommonDirectory(), os.path.splitext(os.path.basename(strUnCorrectedEdfMask))[0])
if self.bCorrectTiltMask:
if not isinstance(unCorrectedMask, fabio.edfimage.edfimage):
strUnCorrectedEdfMask = os.path.join(self.getSPDCommonDirectory(),
os.path.splitext(os.path.basename(self.dictGeometry["MaskFile"]))[0] + ".edf")
unCorrectedEdfMask = fabio.edfimage.edfimage(header={}, data=unCorrectedMask.data.astype("float32"))
unCorrectedEdfMask.write(strUnCorrectedEdfMask)
imgConfig = self.getSPDConfig()
mskConfig = " ".join([i for i in imgConfig.split() \
if not (i.startswith("mask_file=") or i.startswith("azim_")\
or i.startswith("flood_file=") or i.startswith("dark_file="))] + ["do_dark=0", "dummy=1"])
worker = SPDworker()
worker.setExecutable(self.getExecutable())
worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-mask.log"))
worker.initialize(mskConfig)
worker.setTimeOut(self.getTimeOut())
dictLog = worker.process("outdir=%s/ %s" % (self.getSPDCommonDirectory(), strUnCorrectedEdfMask))
for oneLog in dictLog:
self.DEBUG("EDPluginSPDCakev1_5.correctMask %s: %s" % (oneLog, dictLog[oneLog]))
worker.kill(gentle=True)
worker = None
npaCorrected = -(fabio.open(basename + ".cor").data + 0.5).astype("int8")
else:
npaCorrected = -(unCorrectedMask.data).astype("int8")
strCorrectedMask = basename + ".msk"
fabio.edfimage.edfimage(data=npaCorrected, header={"Dummy":"-1"}).write(strCorrectedMask)
EDPluginSPDCakev1_5.__dictMask[self.dictGeometry["MaskFile"]] = strCorrectedMask
EDPluginSPDCakev1_5.__semaphore.release()
newConfig = []
for key in self.getSPDConfig().split():
if key.startswith("mask_file="):
newConfig.append('mask_file=%s' % os.path.abspath(EDPluginSPDCakev1_5.__dictMask[self.dictGeometry["MaskFile"]]))
else:
newConfig.append(key)
self.setSPDConfig(" ".join(newConfig))
# self.worker.setConfig(self.getSPDConfig())
self.DEBUG("%s updating mask %s took %.3fs " % (self.getId(), self.dictGeometry["MaskFile"], time.time() - t0))
class EDPluginSPDCorrectv10(EDPluginExecProcess):
"""
The purpose of this execution plugin is to use SPD to do 2D image correction using a spline and/or a tilt,
Few tips and tricks about SPD ...
* the tilt implementation is done in python and applied to the orthonormmal spline distortion of the image.
"""
__listOfWorker = [] #a dictionary containing the current setup
__iNumberOfWorker = 0 # the max lenth of __listOfWorker
__bInitialized = False #a boolean to tell if it is initialized
__strSPDDir = None # a temporary directory common for SPD
__lockTilt = threading.Semaphore()
__lockWorker = threading.Semaphore()
def __init__(self):
"""
Constructor of the plugin: just do some simple initialization
"""
EDPluginExecProcess.__init__(self)
# self.DEBUG(self.getBaseName())
if self.getClassName() == "EDPluginSPDCorrectv10":
self.setXSDataInputClass(XSDataInputSPD)
self.dictGeometry = {}
self.dictRes = {}
self.xsDataInputSPD = None
self.pathToInputFile = None
self._SPDconfig = None
self.worker = None
self.workerID = None
self._listCompatibleVersions = []
self._strConfigExecutable = None
self.bTimeOut = False
self._iConfigNumberOfWorker = 0
self._bFireAndForget = False
self.strCurrentVersion = None
self.addCompatibleVersion(
"spd version SPD = 1.2 SAXS = 2.436 EDF = 2.171")
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginSPDCorrectv10.checkParameters")
self.checkMandatoryParameters(self.getDataInput(),
"Data Input is None")
self.checkMandatoryParameters(self.getDataInput().getInputFile(),
"inputFile is None")
def preProcess(self, _edObject=None):
"""
Preprocess methods for the EDPluginSPDCorrectv10 :
- Reads input parameters
- Creates the displacement matrix if the the detector is tilted
- create the configuration for SPD
- selects the worker (SPD program under control)
"""
EDPluginExecProcess.preProcess(self)
self.DEBUG("EDPluginSPDCorrectv10.preProcess")
# Check that the input data and correction images are present
self.getInputParameter()
if "SpatialDistortionFile" in self.dictGeometry:
splineDM = Spline()
splineDM.read(self.dictGeometry["SpatialDistortionFile"])
self.dictGeometry["PixelSizeX"], self.dictGeometry[
"PixelSizeY"] = splineDM.getPixelSize()
else:
splineDM = None
if self.dictGeometry["AngleOfTilt"] != 0:
EDPluginSPDCorrectv10.__lockTilt.acquire()
if splineDM == None:
edfFile = fabio.open(self.pathToInputFile)
data = edfFile.data
size = data.shape
splineDM = splineDM.zeros(xmin=0.0,
ymin=0.0,
xmax=size[0],
ymax=size[1])
if ("PixelSizeX"
in self.dictGeometry) and ("PixelSizeY"
in self.dictGeometry):
splineDM.setPixelSize = (self.dictGeometry["PixelSizeX"],
self.dictGeometry["PixelSizeY"])
strtmp = os.path.join(
self.getSPDCommonDirectory(),
os.path.basename(
os.path.splitext(
self.dictGeometry["SpatialDistortionFile"])[0]))
if not (os.path.isfile(strtmp + "-tilted-x.edf")
and os.path.isfile(strtmp + "-tilted-y.edf")):
# self.DEBUG("preProcess: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
if not (os.path.isfile(strtmp + "-tilted-x.edf")
and os.path.isfile(strtmp + "-tilted-y.edf")):
#The second test is just here to gain some time as the global semaphore could be in use elsewhere
self.createDisplacementMatrix(splineDM)
self.dictGeometry["DistortionFileX"] = strtmp + "-tilted-x.edf"
self.dictGeometry["DistortionFileY"] = strtmp + "-tilted-y.edf"
EDPluginSPDCorrectv10.__lockTilt.release()
self.generateSPDCommand()
def process(self, _edObject=None):
"""
Processing of one image: send the image to process to the good worker
"""
self.DEBUG("EDPluginSPDCorrectv1_0.process starting")
self.chooseWorker()
self.synchronizeOn()
self.DEBUG(self.getBaseName() + ": Processing")
if self.isVerboseDebug():
self.DEBUG("EDPluginSPDCorrectv1_0.process worker number: %s " %
self.workerID)
self.DEBUG("EDPluginSPDCorrectv1_0.process Logs will be in %s" %
self.worker.getLogFilename())
if self._bFireAndForget:
if self.dictGeometry["OutputDir"].endswith("/"):
strCmdLine = "outdir=%s %s" % (self.dictGeometry["OutputDir"],
self.pathToInputFile)
else:
strCmdLine = " outdir=%s/ %s" % (
self.dictGeometry["OutputDir"], self.pathToInputFile)
else:
strCmdLine = self.pathToInputFile
self.dictRes = self.worker.process(strCmdLine)
if (self.dictRes["timeout"] == True):
self.DEBUG(
" EDPluginSPDCorrectv1_0.process ========================================= TIMEOUT ================"
)
errorMessage = 'EDPluginSPDCorrectv1_0.process called as %s-%s : TIMEOUT!' % (
self.getClassName(), self.getId())
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
if (self.dictRes["error"] == True):
self.DEBUG(
"EDPluginSPDCorrectv1_0.process ========================================= Execution Error! ================"
)
errorMessage = 'EDPluginSPDCorrectv1_0.process called as %s-%s : Execution Error !' % (
self.getClassName(), self.getId())
# for mesg in self.dictRes:
# self.error("%s: %s" % (mesg, self.dictRes[mesg]))
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
# self.DEBUG("dictRes= %s" % self.dictRes)
self.DEBUG("EDPluginSPDCorrectv1_0.process finished ")
self.synchronizeOff()
def postProcess(self, _edObject=None):
"""
postProcess of the plugin EDPluginSPDCorrectv10.py:
- convert to HDF if needed (to be implemented)
- move images (if needed)
- set result XML
"""
EDPluginExecProcess.postProcess(self)
self.DEBUG("EDPluginSPDCorrectv10.postProcess")
EDUtilsPath.createFolder(self.dictGeometry["OutputDir"])
if self.getClassName() == "EDPluginSPDCorrectv10":
strInputImagePathNoSfx = os.path.splitext(
os.path.basename(self.pathToInputFile))[0]
destFileBaseName = strInputImagePathNoSfx + self.dictGeometry[
"OutputFileType"]
strOutputFilePath = os.path.join(self.dictGeometry["OutputDir"],
destFileBaseName)
if not self._bFireAndForget:
if "corrected" in self.dictRes:
strTempFilePath = self.dictRes["corrected"]
else:
strTempFilePath = os.path.join(self.getWorkingDirectory(),
destFileBaseName)
if self.dictGeometry["OutputFileType"].lower() in [
".hdf5", ".nexus", ".h5", ".nx"
]:
self.WARNING(
"HDF5/Nexus output is not yet implemented in the SPD plugin."
)
if os.path.exists(strOutputFilePath):
self.WARNING(
"Destination file exists, I will leave result file in %s."
% strTempFilePath)
strOutputFilePath = strTempFilePath
else:
shutil.move(strTempFilePath, strOutputFilePath)
# # Create the output data
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(strOutputFilePath))
xsDataResultSPD = XSDataResultSPD()
xsDataResultSPD.setCorrectedFile(xsDataFile)
self.setDataOutput(xsDataResultSPD)
def getInputParameter(self, _edObject=None):
"""
Read all the input parameters and store them in instance variables called self.dictGeometry and self.pathToInputFile
"""
self.DEBUG("EDPluginSPDCorrectv10.getInputParameter")
self.xsDataInputSPD = self.getDataInput()
self.pathToInputFile = self.xsDataInputSPD.getInputFile().getPath(
).getValue()
if not os.path.isfile(self.pathToInputFile):
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess", self.pathToInputFile)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getDarkCurrentImageFile() is not None:
pathToDarkCurrentImageFile = self.xsDataInputSPD.getDarkCurrentImageFile(
).getPath().getValue()
if os.path.isfile(pathToDarkCurrentImageFile):
self.dictGeometry[
"DarkCurrentImageFile"] = pathToDarkCurrentImageFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDarkCurrentImageFile)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getFlatFieldImageFile() is not None:
pathToFlatFieldImageFile = self.xsDataInputSPD.getFlatFieldImageFile(
).getPath().getValue()
if os.path.isfile(pathToFlatFieldImageFile):
self.dictGeometry[
"FlatFieldImageFile"] = pathToFlatFieldImageFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToFlatFieldImageFile)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getSpatialDistortionFile() is not None:
pathToSpatialDistortionFile = self.xsDataInputSPD.getSpatialDistortionFile(
).getPath().getValue()
if os.path.isfile(pathToSpatialDistortionFile):
self.dictGeometry[
"SpatialDistortionFile"] = pathToSpatialDistortionFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToSpatialDistortionFile)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getAngleOfTilt() is not None:
if self.xsDataInputSPD.getAngleOfTilt().getUnit() is not None:
self.dictGeometry["AngleOfTilt"] = EDUtilsUnit.getValue(
self.xsDataInputSPD.getAngleOfTilt(), "deg")
else: #Fall-back by default to deg
self.WARNING(
"You did not specify the AngleOfTilt unit ... I Guess it is deg"
)
self.dictGeometry[
"AngleOfTilt"] = self.xsDataInputSPD.getAngleOfTilt(
).getValue()
else:
self.dictGeometry["AngleOfTilt"] = 0
if self.xsDataInputSPD.getTiltRotation() is not None:
if self.xsDataInputSPD.getTiltRotation().getUnit() is not None:
self.dictGeometry["TiltRotation"] = EDUtilsUnit.getValue(
self.xsDataInputSPD.getTiltRotation(), "deg")
else: #Fall-back by default to deg
self.WARNING(
"You did not specify the TiltRotation unit ... I Guess it is deg"
)
self.dictGeometry[
"TiltRotation"] = self.xsDataInputSPD.getTiltRotation(
).getValue()
else:
self.dictGeometry["TiltRotation"] = 0
if self.xsDataInputSPD.getBeamCentreInPixelsX() is not None:
self.dictGeometry[
"BeamCenterX"] = self.xsDataInputSPD.getBeamCentreInPixelsX(
).getValue()
if self.xsDataInputSPD.getBeamCentreInPixelsY() is not None:
self.dictGeometry[
"BeamCenterY"] = self.xsDataInputSPD.getBeamCentreInPixelsY(
).getValue()
if self.xsDataInputSPD.getBufferSizeX() is not None:
self.dictGeometry[
"BufferSizeX"] = self.xsDataInputSPD.getBufferSizeX().getValue(
)
if self.xsDataInputSPD.getBufferSizeY() is not None:
self.dictGeometry[
"BufferSizeY"] = self.xsDataInputSPD.getBufferSizeY().getValue(
)
if self.xsDataInputSPD.getDistortionFileX() is not None:
pathToDistortionFileX = self.xsDataInputSPD.getDistortionFileX(
).getPath().getValue()
if os.path.isfile(pathToDistortionFileX):
self.dictGeometry["DistortionFileX"] = pathToDistortionFileX
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDistortionFileX)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getDistortionFileY() is not None:
pathToDistortionFileY = self.xsDataInputSPD.getDistortionFileY(
).getPath().getValue()
if os.path.isfile(pathToDistortionFileY):
self.dictGeometry["DistortionFileY"] = pathToDistortionFileY
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDistortionFileY)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getPixelSizeX() is not None:
self.dictGeometry["PixelSizeX"] = EDUtilsUnit.getSIValue(
self.xsDataInputSPD.getPixelSizeX())
if self.xsDataInputSPD.getPixelSizeY() is not None:
self.dictGeometry["PixelSizeY"] = EDUtilsUnit.getSIValue(
self.xsDataInputSPD.getPixelSizeY())
if self.xsDataInputSPD.getSampleToDetectorDistance() is not None:
self.dictGeometry[
"SampleToDetectorDistance"] = EDUtilsUnit.getSIValue(
self.xsDataInputSPD.getSampleToDetectorDistance())
if self.xsDataInputSPD.getWavelength() is not None:
if self.xsDataInputSPD.getWavelength().getUnit() is not None:
self.dictGeometry["Wavelength"] = EDUtilsUnit.getSIValue(
self.xsDataInputSPD.getWavelength())
else: #Fall-back to Angsrom
self.WARNING(
"You did not specify the wavelength unit ... I Guess it is Angstrom"
)
self.dictGeometry[
"Wavelength"] = self.xsDataInputSPD.getWavelength(
).getValue() * 1e-10
if self.xsDataInputSPD.getOutputFileType() is not None:
self.dictGeometry[
"OutputFileType"] = self.xsDataInputSPD.getOutputFileType(
).getValue()
if not self.dictGeometry["OutputFileType"].startswith("."):
self.dictGeometry["OutputFileType"] = "." + self.dictGeometry[
"OutputFileType"]
else:
self.dictGeometry["OutputFileType"] = ".edf"
if self.xsDataInputSPD.getOutputDir() is not None:
tmpdir = self.xsDataInputSPD.getOutputDir().getPath().getValue()
if os.path.isdir(tmpdir):
self.dictGeometry["OutputDir"] = tmpdir
if tmpdir != os.path.dirname(self.pathToInputFile):
self._bFireAndForget = True
else:
self.WARNING(
"The destination directory %s does not exist, using working directory instead %s"
% (tmpdir, self.getWorkingDirectory()))
self.dictGeometry["OutputDir"] = self.getWorkingDirectory()
else:
self.dictGeometry["OutputDir"] = os.path.dirname(
self.pathToInputFile)
if self.dictGeometry["OutputDir"] == os.path.dirname(self.pathToInputFile) and \
not (self.pathToInputFile.endswith(self.dictGeometry["OutputFileType"])) and \
not (self.dictGeometry["OutputFileType"].lower() in [".h5", ".chi", ".nx", ".nexus", ".cif"]) :
self.bFireAndForget = True
def createDisplacementMatrix(self, splineDM):
"""
generate the displacement matrix for SPD
@param splineDM: displacement matrix of a non tilted detector.
@type splineDM: instance of class Spline
"""
splineDM.spline2array()
tilted = splineDM.tilt((self.dictGeometry["BeamCenterX"], self.dictGeometry["BeamCenterY"]), \
self.dictGeometry["AngleOfTilt"], self.dictGeometry["TiltRotation"], self.dictGeometry["SampleToDetectorDistance"])
strtmp = os.path.join(
self.getSPDCommonDirectory(),
os.path.splitext(
os.path.basename(
self.dictGeometry["SpatialDistortionFile"]))[0])
tilted.writeEDF(strtmp + "-tilted")
def generateSPDCommand(self):
"""
This method creates the SPD command line for image correction.
"""
self.DEBUG(" EDPluginSPDCorrectv10.generateSPDCommand")
lstCmdLineOption = ["off_1=0 off_2=0"]
if self.isVerboseDebug() is True:
lstCmdLineOption.append("verbose=2")
else:
lstCmdLineOption.append("verbose=0")
lstCmdLineOption.append('src_ext=%s' % (os.path.splitext(
self.xsDataInputSPD.getInputFile().getPath().getValue())[1]))
if ("OutputFileType"
in self.dictGeometry) and (self.getClassName()
== "EDPluginSPDCorrectv10"):
if self.dictGeometry["OutputFileType"].startswith("."):
lstCmdLineOption.append("cor_ext=%s" %
self.dictGeometry["OutputFileType"])
else:
lstCmdLineOption.append("cor_ext=.%s" %
self.dictGeometry["OutputFileType"])
if "Wavelength" in self.dictGeometry:
lstCmdLineOption.append("wvl=%e" %
(self.dictGeometry["Wavelength"]))
if ("BeamCenterX" in self.dictGeometry) and ("BeamCenterY"
in self.dictGeometry):
lstCmdLineOption.append("cen_1=%e cen_2=%e" %
(self.dictGeometry["BeamCenterX"],
self.dictGeometry["BeamCenterY"]))
if ("SampleToDetectorDistance" in self.dictGeometry):
lstCmdLineOption.append(
"dis=%e" % (self.dictGeometry["SampleToDetectorDistance"]))
if ("PixelSizeX" in self.dictGeometry) and ("PixelSizeY"
in self.dictGeometry):
lstCmdLineOption.append("pix_1=%e pix_2=%e" %
(self.dictGeometry["PixelSizeX"],
self.dictGeometry["PixelSizeY"]))
if ("DistortionFileX" in self.dictGeometry) and ("DistortionFileY"
in self.dictGeometry):
lstCmdLineOption.append('do_distortion=2 xfile=%s yfile=%s' %
(self.dictGeometry["DistortionFileX"],
self.dictGeometry["DistortionFileY"]))
elif ("SpatialDistortionFile" in self.dictGeometry):
lstCmdLineOption.append('do_distortion=2 distortion_file=%s' %
self.dictGeometry["SpatialDistortionFile"])
else:
lstCmdLineOption.append("do_distortion=0")
if ("DarkCurrentImageFile" in self.dictGeometry):
lstCmdLineOption.append('dark_file=%s' %
self.dictGeometry["DarkCurrentImageFile"])
else:
lstCmdLineOption.append("do_dark=0")
if ("FlatFieldImageFile" in self.dictGeometry):
lstCmdLineOption.append('flood_file=%s' %
self.dictGeometry["FlatFieldImageFile"])
strCmdLineOption = " ".join(lstCmdLineOption)
# for i in lstCmdLineOption: self.screen(i)
if "EDPluginSPDCorrectv10" in self.getName():
self.DEBUG("SPD Command line:\n" + strCmdLineOption)
self.setSPDConfig(strCmdLineOption)
def configure(self):
"""
Configures the plugin from the configuration file with the following parameters
- Script executable to be invoked
- number of worker
- The 3rd party executable installed version
"""
EDPluginExecProcess.configure(self)
self.DEBUG(" EDPluginSPDCorrect.configure")
xsPluginItem = self.getConfiguration()
if (xsPluginItem == None):
self.WARNING(
"EDPluginSPDCorrect.configure: No plugin item defined.")
xsPluginItem = XSPluginItem()
if (self.getExecutable() is None):
edStringScriptExecutable = EDConfiguration.getStringParamValue(
xsPluginItem, CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE)
if (edStringScriptExecutable == None):
errorMessage = EDMessage.ERROR_EXECUTION_03 % ('EDPluginSPDCorrect.process', self.getClassName(), "Configuration parameter missing: " \
+ CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
else:
# Check that the executable file exists
if not os.path.isfile(edStringScriptExecutable):
errorMessage = EDMessage.ERROR_EXECUTION_03 % ('EDPluginSPDCorrect.process', self.getClassName(), "Cannot find configured " \
+ CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE + " : " + edStringScriptExecutable)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
else:
self.setExecutable(edStringScriptExecutable)
edStringConfigSetupNbWorker = EDConfiguration.getStringParamValue(
xsPluginItem, CONF_EXEC_MAX_MAX_NUMBER_OF_WORKERS)
if (edStringConfigSetupNbWorker == None):
self.DEBUG(
"EDPluginSPDCorrect.configure: No configuration parameter found for: "
+ CONF_EXEC_MAX_MAX_NUMBER_OF_WORKERS + ", NO default value!")
else:
self._iConfigNumberOfWorker = int(edStringConfigSetupNbWorker)
edStringVersion = EDConfiguration.getStringParamValue(
xsPluginItem, CONF_EXEC_PROCESS_SCRIPT_VERSION_STRING)
if (edStringVersion == None):
self.DEBUG(
"EDPluginSPDCorrect.configure: No configuration parameter found for: "
+ CONF_EXEC_PROCESS_SCRIPT_VERSION_STRING +
", NO default value!")
else:
self.setStringVersion(edStringVersion)
def chooseWorker(self):
"""
selects the worker and sets self.worker
"""
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(
self._iConfigNumberOfWorker)
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s"
% (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker,
EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
# self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
EDPluginSPDCorrectv10.__lockWorker.acquire()
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker:
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s"
% (self.getId(), oneWorker.pid, oneWorker.getStatus()))
if (oneWorker.getConfig()
== self._SPDconfig) and (oneWorker.getStatus()
in ["free", "uninitialized"]):
self.worker = oneWorker
if self.worker is None:
if len(EDPluginSPDCorrectv10.__listOfWorker
) < EDPluginSPDCorrectv10.__iNumberOfWorker:
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker"
%
(self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
self.worker = SPDworker()
EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
self.worker.setExecutable(self.getExecutable())
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(
self.worker)
self.worker.setLogFilename(
os.path.join(self.getSPDCommonDirectory(),
"worker-%02i.log" % self.workerID))
self.worker.initialize(self._SPDconfig)
self.worker.setTimeOut(self.getTimeOut())
while self.worker is None:
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize"
% self.getId())
bConfigOK = False
for idx, oneWorker in enumerate(
EDPluginSPDCorrectv10.__listOfWorker):
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
#bug1 those string must specifically be the same ... change line 248 in spdCake1.5
if (oneWorker.getConfig() == self._SPDconfig):
bConfigOK = True
if (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
self.workerID = idx
if bConfigOK == False:
for idx, oneWorker in enumerate(
EDPluginSPDCorrectv10.__listOfWorker[:]):
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
if (oneWorker.getStatus() in ["free", "uninitialized"]):
#Bug2 why doest this work ???
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
oneWorker.initialize(self._SPDconfig)
self.worker = oneWorker
self.workerID = idx
EDPluginSPDCorrectv10.__listOfWorker[idx] = (
self.worker)
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s "
% (self.getId(), self.workerID, oneWorker))
time.sleep(0.1)
if self.workerID is None:
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(
self.worker)
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" %
(self.getId(),
EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
EDPluginSPDCorrectv10.__lockWorker.release()
@staticmethod
def killAllWorkers():
"""
this methods finishes the process of all workers process underneath
"""
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker:
oneWorker.kill(gentle=True)
oneWorker.closeLogFile()
@staticmethod
def cleanDispMat(_path=None):
"""
This is to ensure that the displacement matrices are removed between 2 tests,
if _path is given, try to backup file in this directory
@param _path: path to backu file to
@type _path: python string representing a path.
"""
for onefile in os.listdir(
EDPluginSPDCorrectv10.getSPDCommonDirectory()):
if onefile.endswith(".edf"):
myFile = os.path.join(
EDPluginSPDCorrectv10.getSPDCommonDirectory(), onefile)
if _path is not None:
try:
shutil.move(myFile, os.path.join(_path, onefile))
except IOError:
EDVerbose.WARNING(
"Problems in moving EDF displacement matrices from %s to %s"
% (myFile, _path))
else:
try:
os.remove(myFile)
except OSError:
EDVerbose.WARNING(
"Problems in removing EDF displacement matrices: %s"
% myFile)
@staticmethod
def getSPDCommonDirectory():
"""
Create a temporary directory common for all SPD process / plugins in order to store displacement matrices
@return: the name of the common directory
@rtype: string
"""
if EDPluginSPDCorrectv10.__strSPDDir is None:
EDPluginSPDCorrectv10.__strSPDDir = tempfile.mkdtemp(
suffix='.tmp', prefix='edna-SPD-')
EDVerbose.screen("The SPD-EDNA logs and matrices will be in " +
EDPluginSPDCorrectv10.__strSPDDir)
return EDPluginSPDCorrectv10.__strSPDDir
def setSPDConfig(self, strConfig):
"""
Setter for the configuration of SPD stored as a command line string
@param strConfig: the configuration of SPD (without the input filename)
@type strConfig: string
"""
self._SPDconfig = strConfig
def getSPDConfig(self):
"""
Getter for the configuration of SPD stored as a command line string
@return: the configuration of SPD (without the input filename)
@rtype strConfig: string
"""
if self._SPDconfig == None:
self.generateSPDCommand()
return self._SPDconfig
def setExecutable(self, _edStringExecutable):
"""
Sets the executable
"""
self.synchronizeOn()
self._strConfigExecutable = _edStringExecutable
self.synchronizeOff()
def getExecutable(self):
"""
Sets the executable
"""
return self._strConfigExecutable
def getListOfCompatibleVersions(self):
"""
Returns the list of compatible executable versions the plugin supports
"""
return self._listCompatibleVersions
def addCompatibleVersion(self, strCompatibleVersion):
"""
Adds a compatible executable version to the list
"""
self._listCompatibleVersions.append(strCompatibleVersion)
def getCompatibleVersionsStringLine(self):
"""
This Method constructs a string line by concatening the compatible versions this plugin supports
This is for Log message purpose only.
"""
strCompatibleVersionsStringLine = ""
for compatibleVersion in self._listCompatibleVersions:
strCompatibleVersionsStringLine += compatibleVersion + " ; "
return strCompatibleVersionsStringLine
def setStringVersion(self, _edStringVersion):
"""
Sets the executable version
"""
self.synchronizeOn()
self.strCurrentVersion = _edStringVersion
self.synchronizeOff()
def getStringVersion(self):
"""
Returns the executable version
"""
returnValue = None
if (self.strCurrentVersion is not None):
returnValue = self.strCurrentVersion
return returnValue
#@property such decorators are not yet available in python2.5 !!!!
def getFireAndForget(self):
return self._bFireAndForget
#@bFireAndForget.setter
def setFireAndForget(self, _bFireNF):
self._bFireAndForget = _bFireNF
bFireAndForget = property(getFireAndForget, setFireAndForget)
class EDPluginSPDCorrectv10(EDPluginExecProcess):
"""
The purpose of this execution plugin is to use SPD to do 2D image correction using a spline and/or a tilt,
Few tips and tricks about SPD ...
* the tilt implementation is done in python and applied to the orthonormmal spline distortion of the image.
"""
__listOfWorker = [] # a dictionary containing the current setup
__iNumberOfWorker = 0 # the max lenth of __listOfWorker
__bInitialized = False # a boolean to tell if it is initialized
__strSPDDir = None # a temporary directory common for SPD
__lockTilt = threading.Semaphore()
__lockWorker = threading.Semaphore()
def __init__(self):
"""
Constructor of the plugin: just do some simple initialization
"""
EDPluginExecProcess.__init__(self)
# self.DEBUG(self.getBaseName())
if self.getClassName() == "EDPluginSPDCorrectv10":
self.setXSDataInputClass(XSDataInputSPD)
self.dictGeometry = {}
self.dictRes = {}
self.xsDataInputSPD = None
self.pathToInputFile = None
self._SPDconfig = None
self.worker = None
self.workerID = None
self._listCompatibleVersions = []
self._strConfigExecutable = None
self.bTimeOut = False
self._iConfigNumberOfWorker = 0
self._bFireAndForget = False
self.strCurrentVersion = None
self.addCompatibleVersion("spd version SPD = 1.2 SAXS = 2.436 EDF = 2.171")
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
self.DEBUG("EDPluginSPDCorrectv10.checkParameters")
self.checkMandatoryParameters(self.getDataInput(), "Data Input is None")
self.checkMandatoryParameters(self.getDataInput().getInputFile(), "inputFile is None")
def preProcess(self, _edObject=None):
"""
Preprocess methods for the EDPluginSPDCorrectv10 :
- Reads input parameters
- Creates the displacement matrix if the the detector is tilted
- create the configuration for SPD
- selects the worker (SPD program under control)
"""
EDPluginExecProcess.preProcess(self)
self.DEBUG("EDPluginSPDCorrectv10.preProcess")
# Check that the input data and correction images are present
self.getInputParameter()
if "SpatialDistortionFile" in self.dictGeometry:
splineDM = Spline()
splineDM.read(self.dictGeometry["SpatialDistortionFile"])
self.dictGeometry["PixelSizeX"], self.dictGeometry["PixelSizeY"] = splineDM.getPixelSize()
else:
splineDM = None
if self.dictGeometry["AngleOfTilt"] != 0:
EDPluginSPDCorrectv10.__lockTilt.acquire()
if splineDM == None:
edfFile = fabio.open(self.pathToInputFile)
data = edfFile.data
size = data.shape
splineDM = splineDM.zeros(xmin=0.0, ymin=0.0, xmax=size[0], ymax=size[1])
if ("PixelSizeX" in self.dictGeometry) and ("PixelSizeY" in self.dictGeometry):
splineDM.setPixelSize = (self.dictGeometry["PixelSizeX"], self.dictGeometry["PixelSizeY"])
strtmp = os.path.join(
self.getSPDCommonDirectory(),
os.path.basename(os.path.splitext(self.dictGeometry["SpatialDistortionFile"])[0]),
)
if not (os.path.isfile(strtmp + "-tilted-x.edf") and os.path.isfile(strtmp + "-tilted-y.edf")):
# self.DEBUG("preProcess: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
if not (os.path.isfile(strtmp + "-tilted-x.edf") and os.path.isfile(strtmp + "-tilted-y.edf")):
# The second test is just here to gain some time as the global semaphore could be in use elsewhere
self.createDisplacementMatrix(splineDM)
self.dictGeometry["DistortionFileX"] = strtmp + "-tilted-x.edf"
self.dictGeometry["DistortionFileY"] = strtmp + "-tilted-y.edf"
EDPluginSPDCorrectv10.__lockTilt.release()
self.generateSPDCommand()
def process(self, _edObject=None):
"""
Processing of one image: send the image to process to the good worker
"""
self.DEBUG("EDPluginSPDCorrectv1_0.process starting")
self.chooseWorker()
self.synchronizeOn()
self.DEBUG(self.getBaseName() + ": Processing")
if self.isVerboseDebug():
self.DEBUG("EDPluginSPDCorrectv1_0.process worker number: %s " % self.workerID)
self.DEBUG("EDPluginSPDCorrectv1_0.process Logs will be in %s" % self.worker.getLogFilename())
if self._bFireAndForget:
if self.dictGeometry["OutputDir"].endswith("/"):
strCmdLine = "outdir=%s %s" % (self.dictGeometry["OutputDir"], self.pathToInputFile)
else:
strCmdLine = " outdir=%s/ %s" % (self.dictGeometry["OutputDir"], self.pathToInputFile)
else:
strCmdLine = self.pathToInputFile
self.dictRes = self.worker.process(strCmdLine)
if self.dictRes["timeout"] == True:
self.DEBUG(
" EDPluginSPDCorrectv1_0.process ========================================= TIMEOUT ================"
)
errorMessage = "EDPluginSPDCorrectv1_0.process called as %s-%s : TIMEOUT!" % (
self.getClassName(),
self.getId(),
)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
if self.dictRes["error"] == True:
self.DEBUG(
"EDPluginSPDCorrectv1_0.process ========================================= Execution Error! ================"
)
errorMessage = "EDPluginSPDCorrectv1_0.process called as %s-%s : Execution Error !" % (
self.getClassName(),
self.getId(),
)
# for mesg in self.dictRes:
# self.error("%s: %s" % (mesg, self.dictRes[mesg]))
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
# self.DEBUG("dictRes= %s" % self.dictRes)
self.DEBUG("EDPluginSPDCorrectv1_0.process finished ")
self.synchronizeOff()
def postProcess(self, _edObject=None):
"""
postProcess of the plugin EDPluginSPDCorrectv10.py:
- convert to HDF if needed (to be implemented)
- move images (if needed)
- set result XML
"""
EDPluginExecProcess.postProcess(self)
self.DEBUG("EDPluginSPDCorrectv10.postProcess")
EDUtilsPath.createFolder(self.dictGeometry["OutputDir"])
if self.getClassName() == "EDPluginSPDCorrectv10":
strInputImagePathNoSfx = os.path.splitext(os.path.basename(self.pathToInputFile))[0]
destFileBaseName = strInputImagePathNoSfx + self.dictGeometry["OutputFileType"]
strOutputFilePath = os.path.join(self.dictGeometry["OutputDir"], destFileBaseName)
if not self._bFireAndForget:
if "corrected" in self.dictRes:
strTempFilePath = self.dictRes["corrected"]
else:
strTempFilePath = os.path.join(self.getWorkingDirectory(), destFileBaseName)
if self.dictGeometry["OutputFileType"].lower() in [".hdf5", ".nexus", ".h5", ".nx"]:
self.WARNING("HDF5/Nexus output is not yet implemented in the SPD plugin.")
if os.path.exists(strOutputFilePath):
self.WARNING("Destination file exists, I will leave result file in %s." % strTempFilePath)
strOutputFilePath = strTempFilePath
else:
shutil.move(strTempFilePath, strOutputFilePath)
# # Create the output data
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(strOutputFilePath))
xsDataResultSPD = XSDataResultSPD()
xsDataResultSPD.setCorrectedFile(xsDataFile)
self.setDataOutput(xsDataResultSPD)
def getInputParameter(self, _edObject=None):
"""
Read all the input parameters and store them in instance variables called self.dictGeometry and self.pathToInputFile
"""
self.DEBUG("EDPluginSPDCorrectv10.getInputParameter")
self.xsDataInputSPD = self.getDataInput()
self.pathToInputFile = self.xsDataInputSPD.getInputFile().getPath().getValue()
if not os.path.isfile(self.pathToInputFile):
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
self.pathToInputFile,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getDarkCurrentImageFile() is not None:
pathToDarkCurrentImageFile = self.xsDataInputSPD.getDarkCurrentImageFile().getPath().getValue()
if os.path.isfile(pathToDarkCurrentImageFile):
self.dictGeometry["DarkCurrentImageFile"] = pathToDarkCurrentImageFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDarkCurrentImageFile,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getFlatFieldImageFile() is not None:
pathToFlatFieldImageFile = self.xsDataInputSPD.getFlatFieldImageFile().getPath().getValue()
if os.path.isfile(pathToFlatFieldImageFile):
self.dictGeometry["FlatFieldImageFile"] = pathToFlatFieldImageFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToFlatFieldImageFile,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getSpatialDistortionFile() is not None:
pathToSpatialDistortionFile = self.xsDataInputSPD.getSpatialDistortionFile().getPath().getValue()
if os.path.isfile(pathToSpatialDistortionFile):
self.dictGeometry["SpatialDistortionFile"] = pathToSpatialDistortionFile
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToSpatialDistortionFile,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getAngleOfTilt() is not None:
if self.xsDataInputSPD.getAngleOfTilt().getUnit() is not None:
self.dictGeometry["AngleOfTilt"] = EDUtilsUnit.getValue(self.xsDataInputSPD.getAngleOfTilt(), "deg")
else: # Fall-back by default to deg
self.WARNING("You did not specify the AngleOfTilt unit ... I Guess it is deg")
self.dictGeometry["AngleOfTilt"] = self.xsDataInputSPD.getAngleOfTilt().getValue()
else:
self.dictGeometry["AngleOfTilt"] = 0
if self.xsDataInputSPD.getTiltRotation() is not None:
if self.xsDataInputSPD.getTiltRotation().getUnit() is not None:
self.dictGeometry["TiltRotation"] = EDUtilsUnit.getValue(self.xsDataInputSPD.getTiltRotation(), "deg")
else: # Fall-back by default to deg
self.WARNING("You did not specify the TiltRotation unit ... I Guess it is deg")
self.dictGeometry["TiltRotation"] = self.xsDataInputSPD.getTiltRotation().getValue()
else:
self.dictGeometry["TiltRotation"] = 0
if self.xsDataInputSPD.getBeamCentreInPixelsX() is not None:
self.dictGeometry["BeamCenterX"] = self.xsDataInputSPD.getBeamCentreInPixelsX().getValue()
if self.xsDataInputSPD.getBeamCentreInPixelsY() is not None:
self.dictGeometry["BeamCenterY"] = self.xsDataInputSPD.getBeamCentreInPixelsY().getValue()
if self.xsDataInputSPD.getBufferSizeX() is not None:
self.dictGeometry["BufferSizeX"] = self.xsDataInputSPD.getBufferSizeX().getValue()
if self.xsDataInputSPD.getBufferSizeY() is not None:
self.dictGeometry["BufferSizeY"] = self.xsDataInputSPD.getBufferSizeY().getValue()
if self.xsDataInputSPD.getDistortionFileX() is not None:
pathToDistortionFileX = self.xsDataInputSPD.getDistortionFileX().getPath().getValue()
if os.path.isfile(pathToDistortionFileX):
self.dictGeometry["DistortionFileX"] = pathToDistortionFileX
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDistortionFileX,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getDistortionFileY() is not None:
pathToDistortionFileY = self.xsDataInputSPD.getDistortionFileY().getPath().getValue()
if os.path.isfile(pathToDistortionFileY):
self.dictGeometry["DistortionFileY"] = pathToDistortionFileY
else:
edStringErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % (
self.getPluginName() + ".preProcess",
pathToDistortionFileY,
)
self.error(edStringErrorMessage)
self.addErrorMessage(edStringErrorMessage)
raise RuntimeError, edStringErrorMessage
if self.xsDataInputSPD.getPixelSizeX() is not None:
self.dictGeometry["PixelSizeX"] = EDUtilsUnit.getSIValue(self.xsDataInputSPD.getPixelSizeX())
if self.xsDataInputSPD.getPixelSizeY() is not None:
self.dictGeometry["PixelSizeY"] = EDUtilsUnit.getSIValue(self.xsDataInputSPD.getPixelSizeY())
if self.xsDataInputSPD.getSampleToDetectorDistance() is not None:
self.dictGeometry["SampleToDetectorDistance"] = EDUtilsUnit.getSIValue(
self.xsDataInputSPD.getSampleToDetectorDistance()
)
if self.xsDataInputSPD.getWavelength() is not None:
if self.xsDataInputSPD.getWavelength().getUnit() is not None:
self.dictGeometry["Wavelength"] = EDUtilsUnit.getSIValue(self.xsDataInputSPD.getWavelength())
else: # Fall-back to Angsrom
self.WARNING("You did not specify the wavelength unit ... I Guess it is Angstrom")
self.dictGeometry["Wavelength"] = self.xsDataInputSPD.getWavelength().getValue() * 1e-10
if self.xsDataInputSPD.getOutputFileType() is not None:
self.dictGeometry["OutputFileType"] = self.xsDataInputSPD.getOutputFileType().getValue()
if not self.dictGeometry["OutputFileType"].startswith("."):
self.dictGeometry["OutputFileType"] = "." + self.dictGeometry["OutputFileType"]
else:
self.dictGeometry["OutputFileType"] = ".edf"
if self.xsDataInputSPD.getOutputDir() is not None:
tmpdir = self.xsDataInputSPD.getOutputDir().getPath().getValue()
if os.path.isdir(tmpdir):
self.dictGeometry["OutputDir"] = tmpdir
if tmpdir != os.path.dirname(self.pathToInputFile):
self._bFireAndForget = True
else:
self.WARNING(
"The destination directory %s does not exist, using working directory instead %s"
% (tmpdir, self.getWorkingDirectory())
)
self.dictGeometry["OutputDir"] = self.getWorkingDirectory()
else:
self.dictGeometry["OutputDir"] = os.path.dirname(self.pathToInputFile)
if (
self.dictGeometry["OutputDir"] == os.path.dirname(self.pathToInputFile)
and not (self.pathToInputFile.endswith(self.dictGeometry["OutputFileType"]))
and not (self.dictGeometry["OutputFileType"].lower() in [".h5", ".chi", ".nx", ".nexus", ".cif"])
):
self.bFireAndForget = True
def createDisplacementMatrix(self, splineDM):
"""
generate the displacement matrix for SPD
@param splineDM: displacement matrix of a non tilted detector.
@type splineDM: instance of class Spline
"""
splineDM.spline2array()
tilted = splineDM.tilt(
(self.dictGeometry["BeamCenterX"], self.dictGeometry["BeamCenterY"]),
self.dictGeometry["AngleOfTilt"],
self.dictGeometry["TiltRotation"],
self.dictGeometry["SampleToDetectorDistance"],
)
strtmp = os.path.join(
self.getSPDCommonDirectory(),
os.path.splitext(os.path.basename(self.dictGeometry["SpatialDistortionFile"]))[0],
)
tilted.writeEDF(strtmp + "-tilted")
def generateSPDCommand(self):
"""
This method creates the SPD command line for image correction.
"""
self.DEBUG(" EDPluginSPDCorrectv10.generateSPDCommand")
lstCmdLineOption = ["off_1=0 off_2=0"]
if self.isVerboseDebug() is True:
lstCmdLineOption.append("verbose=2")
else:
lstCmdLineOption.append("verbose=0")
lstCmdLineOption.append(
"src_ext=%s" % (os.path.splitext(self.xsDataInputSPD.getInputFile().getPath().getValue())[1])
)
if ("OutputFileType" in self.dictGeometry) and (self.getClassName() == "EDPluginSPDCorrectv10"):
if self.dictGeometry["OutputFileType"].startswith("."):
lstCmdLineOption.append("cor_ext=%s" % self.dictGeometry["OutputFileType"])
else:
lstCmdLineOption.append("cor_ext=.%s" % self.dictGeometry["OutputFileType"])
if "Wavelength" in self.dictGeometry:
lstCmdLineOption.append("wvl=%e" % (self.dictGeometry["Wavelength"]))
if ("BeamCenterX" in self.dictGeometry) and ("BeamCenterY" in self.dictGeometry):
lstCmdLineOption.append(
"cen_1=%e cen_2=%e" % (self.dictGeometry["BeamCenterX"], self.dictGeometry["BeamCenterY"])
)
if "SampleToDetectorDistance" in self.dictGeometry:
lstCmdLineOption.append("dis=%e" % (self.dictGeometry["SampleToDetectorDistance"]))
if ("PixelSizeX" in self.dictGeometry) and ("PixelSizeY" in self.dictGeometry):
lstCmdLineOption.append(
"pix_1=%e pix_2=%e" % (self.dictGeometry["PixelSizeX"], self.dictGeometry["PixelSizeY"])
)
if ("DistortionFileX" in self.dictGeometry) and ("DistortionFileY" in self.dictGeometry):
lstCmdLineOption.append(
"do_distortion=2 xfile=%s yfile=%s"
% (self.dictGeometry["DistortionFileX"], self.dictGeometry["DistortionFileY"])
)
elif "SpatialDistortionFile" in self.dictGeometry:
lstCmdLineOption.append("do_distortion=2 distortion_file=%s" % self.dictGeometry["SpatialDistortionFile"])
else:
lstCmdLineOption.append("do_distortion=0")
if "DarkCurrentImageFile" in self.dictGeometry:
lstCmdLineOption.append("dark_file=%s" % self.dictGeometry["DarkCurrentImageFile"])
else:
lstCmdLineOption.append("do_dark=0")
if "FlatFieldImageFile" in self.dictGeometry:
lstCmdLineOption.append("flood_file=%s" % self.dictGeometry["FlatFieldImageFile"])
strCmdLineOption = " ".join(lstCmdLineOption)
# for i in lstCmdLineOption: self.screen(i)
if "EDPluginSPDCorrectv10" in self.getName():
self.DEBUG("SPD Command line:\n" + strCmdLineOption)
self.setSPDConfig(strCmdLineOption)
def configure(self):
"""
Configures the plugin from the configuration file with the following parameters
- Script executable to be invoked
- number of worker
- The 3rd party executable installed version
"""
EDPluginExecProcess.configure(self)
self.DEBUG(" EDPluginSPDCorrect.configure")
xsPluginItem = self.getConfiguration()
if xsPluginItem == None:
self.WARNING("EDPluginSPDCorrect.configure: No plugin item defined.")
xsPluginItem = XSPluginItem()
if self.getExecutable() is None:
edStringScriptExecutable = EDConfiguration.getStringParamValue(
xsPluginItem, CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE
)
if edStringScriptExecutable == None:
errorMessage = EDMessage.ERROR_EXECUTION_03 % (
"EDPluginSPDCorrect.process",
self.getClassName(),
"Configuration parameter missing: " + CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE,
)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
else:
# Check that the executable file exists
if not os.path.isfile(edStringScriptExecutable):
errorMessage = EDMessage.ERROR_EXECUTION_03 % (
"EDPluginSPDCorrect.process",
self.getClassName(),
"Cannot find configured "
+ CONF_EXEC_PROCESS_SCRIPT_EXECUTABLE
+ " : "
+ edStringScriptExecutable,
)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
else:
self.setExecutable(edStringScriptExecutable)
edStringConfigSetupNbWorker = EDConfiguration.getStringParamValue(
xsPluginItem, CONF_EXEC_MAX_MAX_NUMBER_OF_WORKERS
)
if edStringConfigSetupNbWorker == None:
self.DEBUG(
"EDPluginSPDCorrect.configure: No configuration parameter found for: "
+ CONF_EXEC_MAX_MAX_NUMBER_OF_WORKERS
+ ", NO default value!"
)
else:
self._iConfigNumberOfWorker = int(edStringConfigSetupNbWorker)
edStringVersion = EDConfiguration.getStringParamValue(xsPluginItem, CONF_EXEC_PROCESS_SCRIPT_VERSION_STRING)
if edStringVersion == None:
self.DEBUG(
"EDPluginSPDCorrect.configure: No configuration parameter found for: "
+ CONF_EXEC_PROCESS_SCRIPT_VERSION_STRING
+ ", NO default value!"
)
else:
self.setStringVersion(edStringVersion)
def chooseWorker(self):
"""
selects the worker and sets self.worker
"""
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s"
% (
self.getId(),
EDPluginSPDCorrectv10.__iNumberOfWorker,
EDPluginSPDCorrectv10.__lockWorker._Semaphore__value,
)
)
# self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
EDPluginSPDCorrectv10.__lockWorker.acquire()
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker:
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s"
% (self.getId(), oneWorker.pid, oneWorker.getStatus())
)
if (oneWorker.getConfig() == self._SPDconfig) and (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
if self.worker is None:
if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker:
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker"
% (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker))
)
self.worker = SPDworker()
EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
self.worker.setExecutable(self.getExecutable())
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.worker.setLogFilename(
os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID)
)
self.worker.initialize(self._SPDconfig)
self.worker.setTimeOut(self.getTimeOut())
while self.worker is None:
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
bConfigOK = False
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker):
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
# bug1 those string must specifically be the same ... change line 248 in spdCake1.5
if oneWorker.getConfig() == self._SPDconfig:
bConfigOK = True
if oneWorker.getStatus() in ["free", "uninitialized"]:
self.worker = oneWorker
self.workerID = idx
if bConfigOK == False:
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]):
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
if oneWorker.getStatus() in ["free", "uninitialized"]:
# Bug2 why doest this work ???
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
oneWorker.initialize(self._SPDconfig)
self.worker = oneWorker
self.workerID = idx
EDPluginSPDCorrectv10.__listOfWorker[idx] = self.worker
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s "
% (self.getId(), self.workerID, oneWorker)
)
time.sleep(0.1)
if self.workerID is None:
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.DEBUG(
"EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s"
% (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value)
)
EDPluginSPDCorrectv10.__lockWorker.release()
@staticmethod
def killAllWorkers():
"""
this methods finishes the process of all workers process underneath
"""
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker:
oneWorker.kill(gentle=True)
oneWorker.closeLogFile()
@staticmethod
def cleanDispMat(_path=None):
"""
This is to ensure that the displacement matrices are removed between 2 tests,
if _path is given, try to backup file in this directory
@param _path: path to backu file to
@type _path: python string representing a path.
"""
for onefile in os.listdir(EDPluginSPDCorrectv10.getSPDCommonDirectory()):
if onefile.endswith(".edf"):
myFile = os.path.join(EDPluginSPDCorrectv10.getSPDCommonDirectory(), onefile)
if _path is not None:
try:
shutil.move(myFile, os.path.join(_path, onefile))
except IOError:
EDVerbose.WARNING(
"Problems in moving EDF displacement matrices from %s to %s" % (myFile, _path)
)
else:
try:
os.remove(myFile)
except OSError:
EDVerbose.WARNING("Problems in removing EDF displacement matrices: %s" % myFile)
@staticmethod
def getSPDCommonDirectory():
"""
Create a temporary directory common for all SPD process / plugins in order to store displacement matrices
@return: the name of the common directory
@rtype: string
"""
if EDPluginSPDCorrectv10.__strSPDDir is None:
EDPluginSPDCorrectv10.__strSPDDir = tempfile.mkdtemp(suffix=".tmp", prefix="edna-SPD-")
EDVerbose.screen("The SPD-EDNA logs and matrices will be in " + EDPluginSPDCorrectv10.__strSPDDir)
return EDPluginSPDCorrectv10.__strSPDDir
def setSPDConfig(self, strConfig):
"""
Setter for the configuration of SPD stored as a command line string
@param strConfig: the configuration of SPD (without the input filename)
@type strConfig: string
"""
self._SPDconfig = strConfig
def getSPDConfig(self):
"""
Getter for the configuration of SPD stored as a command line string
@return: the configuration of SPD (without the input filename)
@rtype strConfig: string
"""
if self._SPDconfig == None:
self.generateSPDCommand()
return self._SPDconfig
def setExecutable(self, _edStringExecutable):
"""
Sets the executable
"""
self.synchronizeOn()
self._strConfigExecutable = _edStringExecutable
self.synchronizeOff()
def getExecutable(self):
"""
Sets the executable
"""
return self._strConfigExecutable
def getListOfCompatibleVersions(self):
"""
Returns the list of compatible executable versions the plugin supports
"""
return self._listCompatibleVersions
def addCompatibleVersion(self, strCompatibleVersion):
"""
Adds a compatible executable version to the list
"""
self._listCompatibleVersions.append(strCompatibleVersion)
def getCompatibleVersionsStringLine(self):
"""
This Method constructs a string line by concatening the compatible versions this plugin supports
This is for Log message purpose only.
"""
strCompatibleVersionsStringLine = ""
for compatibleVersion in self._listCompatibleVersions:
strCompatibleVersionsStringLine += compatibleVersion + " ; "
return strCompatibleVersionsStringLine
def setStringVersion(self, _edStringVersion):
"""
Sets the executable version
"""
self.synchronizeOn()
self.strCurrentVersion = _edStringVersion
self.synchronizeOff()
def getStringVersion(self):
"""
Returns the executable version
"""
returnValue = None
if self.strCurrentVersion is not None:
returnValue = self.strCurrentVersion
return returnValue
# @property such decorators are not yet available in python2.5 !!!!
def getFireAndForget(self):
return self._bFireAndForget
# @bFireAndForget.setter
def setFireAndForget(self, _bFireNF):
self._bFireAndForget = _bFireNF
bFireAndForget = property(getFireAndForget, setFireAndForget)
