def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'fileNameParameters':
obj_ = XSDataGridScreeningFileNameParameters()
obj_.build(child_)
self.setFileNameParameters(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'imageQualityIndicators':
obj_ = XSDataImageQualityIndicators()
obj_.build(child_)
self.setImageQualityIndicators(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'mosaicity':
obj_ = XSDataDouble()
obj_.build(child_)
self.setMosaicity(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'rankingResolution':
obj_ = XSDataDouble()
obj_.build(child_)
self.setRankingResolution(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'comment':
obj_ = XSDataString()
obj_.build(child_)
self.setComment(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'imageQualityIndicatorsId':
obj_ = XSDataInteger()
obj_.build(child_)
self.setImageQualityIndicatorsId(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'resultIntegration':
obj_ = XSDataGridScreeningResultIntegration()
obj_.build(child_)
self.setResultIntegration(obj_)
XSDataResult.buildChildren(self, child_, nodeName_)
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
EDUtilsParallel.initializeNbThread()
# Check batch size
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
self.screen(
"Image quality indicators batch size: {0}".format(batchSize))
# Check if we should do distlSignalStrength:
bDoDistlSignalStrength = True
if self.dataInput.doDistlSignalStrength is not None:
if not self.dataInput.doDistlSignalStrength.value:
bDoDistlSignalStrength = False
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Check if fast mesh (for HDF5)
isFastMesh = False
if self.dataInput.fastMesh:
isFastMesh = self.dataInput.fastMesh.value
# Loop through all the incoming reference images
if len(self.dataInput.image) == 0:
directory = self.dataInput.directory.path.value
template = self.dataInput.template.value
startNo = self.dataInput.startNo.value
endNo = self.dataInput.endNo.value
listXSDataImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = os.path.join(directory, imageName)
xsDataImage = XSDataImage(path=XSDataString(imagePath),
number=XSDataInteger(index))
listXSDataImage.append(xsDataImage)
else:
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPluginDistl = []
listPluginDozor = []
listOfImagesInBatch = []
listOfAllBatches = []
indexBatch = 0
listH5FilePath = []
# Process data in batches
for xsDataImage in listXSDataImage:
listOfImagesInBatch.append(xsDataImage.copy())
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
# Loop over batches
for listOfImagesInBatch in listOfAllBatches:
# First wait for images
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# If Eiger, just wait for the h5 file
if strPathToImage.endswith(".h5"):
h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(
strPathToImage,
batchSize=batchSize,
isFastMesh=isFastMesh)
# print(h5FilePath)
# print(hdf5ImageNumber)
if not h5DataFilePath in listH5FilePath:
self.screen(
"ID30a3 Eiger data, waiting for master and data files..."
)
listH5FilePath.append(h5DataFilePath)
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(h5DataFilePath))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen(
"Waiting for file {0}".format(h5DataFilePath))
self.DEBUG("Wait file timeOut set to %f" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
# hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
time.sleep(1)
if not os.path.exists(h5DataFilePath):
strError = "Time-out while waiting for image %s" % h5DataFilePath
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if not os.path.exists(strPathToImage):
# self.screen("Waiting for file {0}".format(strPathToImage))
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(strPathToImage))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen("Wait file timeOut set to %.0f s" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(strPathToImage):
strError = "Time-out while waiting for image %s" % strPathToImage
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
if not self.isFailure():
strPathToFirstImage = listOfImagesInBatch[0].path.value
if strPathToImage.endswith(".h5"):
indexLoop = 1
continueLoop = True
while continueLoop:
directory = os.path.dirname(strPathToFirstImage)
firstImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[0].path.value)
lastImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
xsDataInputH5ToCBF = XSDataInputH5ToCBF()
xsDataInputH5ToCBF.hdf5File = XSDataFile(
listOfImagesInBatch[0].path)
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
xsDataInputH5ToCBF.startImageNumber = XSDataInteger(
firstImage)
xsDataInputH5ToCBF.endImageNumber = XSDataInteger(
lastImage)
xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(
XSDataString(directory))
edPluginH5ToCBF = self.loadPlugin(
"EDPluginH5ToCBFv1_1")
edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
edPluginH5ToCBF.execute()
edPluginH5ToCBF.synchronize()
outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
if outputCBFFileTemplate is not None:
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:06d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)))
strPathToImage = os.path.join(
directory, lastCbfFile)
# print(cbfFile.path.value)
if os.path.exists(strPathToImage):
# Rename all images
for image in listOfImagesInBatch:
image.path.value = image.path.value.replace(
".h5", ".cbf")
imageNumber = EDUtilsImage.getImageNumber(
image.path.value)
oldPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:06d}".format(imageNumber)))
newPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:04d}".format(imageNumber)))
os.rename(oldPath, newPath)
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:04d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
))
strPathToImage = os.path.join(
directory, lastCbfFile)
self.screen(
"Image has been converted to CBF file: {0}"
.format(strPathToImage))
continueLoop = False
# print(continueLoop)
if continueLoop:
self.screen(
"Still waiting for converting to CBF file: {0}"
.format(strPathToImage))
indexLoop += 1
time.sleep(5)
if indexLoop > 10:
continueLoop = False
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# Check if we should run distl.signalStrength
xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
xsDataImageNew.number = XSDataInteger(
EDUtilsImage.getImageNumber(image.path.value))
edPluginPluginExecImageQualityIndicator = None
if bDoDistlSignalStrength:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(
xsDataImageNew)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listPluginDistl.append(
(xsDataImageNew.copy(),
edPluginPluginExecImageQualityIndicator))
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor, "ControlDozor_{0}".format(
os.path.splitext(
os.path.basename(strPathToFirstImage))[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listOfImagesInBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
len(listOfImagesInBatch))
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append(
(edPluginControlDozor, list(listOfImagesInBatch)))
if not self.isFailure():
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for (xsDataImage,
edPluginPluginExecImageQualityIndicator) in listPluginDistl:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
xsDataImageQualityIndicators.image = xsDataImage.copy()
if edPluginPluginExecImageQualityIndicator is not None:
edPluginPluginExecImageQualityIndicator.synchronize()
if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
for (edPluginControlDozor, listBatch) in listPluginDozor:
edPluginControlDozor.synchronize()
# Check that we got at least one result
if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = os.path.basename(listBatch[0].path.value)
lastImage = os.path.basename(listBatch[-1].path.value)
self.screen(
"No dozor results! Re-executing Dozor for images {0} to {1}"
.format(firstImage, lastImage))
time.sleep(5)
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor,
"ControlDozor_reexecution_{0}".format(
os.path.splitext(firstImage)[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(
image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.executeSynchronous()
for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
if imageDozor.spotFile is not None:
if os.path.exists(
imageDozor.spotFile.path.value):
numpyArray = numpy.loadtxt(
imageDozor.spotFile.path.value,
skiprows=3)
xsDataImageQualityIndicators.dozorSpotList = XSDataString(
base64.b64encode(
numpyArray.tostring()))
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[0]))
if len(numpyArray.shape) > 1:
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[1]))
xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
if edPluginControlDozor.dataOutput.inputDozor is not None:
self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput(
).parseString(
edPluginControlDozor.dataOutput.
inputDozor.marshal())
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have dozor_score from all images:
has_dozor_score = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.dozor_score is None:
has_dozor_score = False
if has_dozor_score:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.dozor_score.value)
else:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[
-5:]:
if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
xsDataInputReadImageHeader = XSDataInputReadImageHeader(
)
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path
)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
edPluginLabelitIndexing = self.loadPlugin(
self.strIndexingLabelitPluginName)
xsDataInputLabelitIndexing = XSDataInputLabelitIndexing(
)
xsDataInputLabelitIndexing.image.append(
XSDataImage(
xsDataResultControlImageQualityIndicator.
image.path))
edPluginLabelitIndexing.setDataInput(
xsDataInputLabelitIndexing)
self.listPluginLabelit.append([
edPluginLabelitIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginLabelitIndexing.execute()
for tupleLabelit in self.listPluginLabelit:
edPluginLabelitIndexing = tupleLabelit[0]
xsDataResultControlImageQualityIndicator = tupleLabelit[1]
edPluginLabelitIndexing.synchronize()
if not edPluginLabelitIndexing.isFailure(
) and edPluginLabelitIndexing.dataOutput is not None:
xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput(
)
xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
xsDataResultLabelitIndexing)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
