def test_execute_ControlDozor_eiger4m(self):
referenceDataPath = self.dataPath / "ControlDozor_eiger4m.json"
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData["imageQualityIndicators"]), 4)
def test_execute_ControlDozor_wedgeNumber(self):
referenceDataPath = self.dataPath / 'ControlDozor_wedgeNumber.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 740)
def test_execute_ControlDozor_ispyb_h5(self):
currentSite = UtilsConfig.getSite()
referenceDataPath = self.dataPath / 'ControlDozor_ispyb_hdf5.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
UtilsConfig.setSite(currentSite)
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 51)
def testCreateListOfBatches(self):
self.assertEqual([[1], [2], [3], [4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 1))
self.assertEqual([[1, 2], [3, 4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 2))
self.assertEqual([[1, 2, 3], [4, 5]],
ControlDozor.createListOfBatches(range(1, 6), 3))
self.assertEqual([[1, 2, 3, 4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 4))
self.assertEqual([[1, 2, 3, 4, 5]],
ControlDozor.createListOfBatches(range(1, 6), 5))
self.assertEqual(
[[1], [2], [4], [5], [6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 1),
)
self.assertEqual(
[[1, 2], [4, 5], [6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 2),
)
self.assertEqual(
[[1, 2], [4, 5, 6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 3),
)
def synchronizeDozor(self, listDozorTask, listDistlResult):
listImageQualityIndicators = []
listControlDozorAllFile = []
for (
controlDozor,
inDataControlDozor,
listBatch,
) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData["imageQualityIndicators"]) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0]
lastImage = listBatch[-1]
logger.warning(
"No dozor results! Re-executing Dozor for"
+ " images {0} to {1}".format(firstImage, lastImage)
)
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(
controlDozor.outData["imageQualityIndicators"]
)
if self.doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor["image"] == distlResult["image"]:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if self.doDozorM:
listControlDozorAllFile.append(controlDozor.outData["dozorAllFile"])
return listImageQualityIndicators, listControlDozorAllFile
def test_makePlot(self):
workingDirectory = UtilsTest.getTestRunPath()
dataCollectionId = 123456
outDataPath = self.dataPath / "outDataControlDozor.json"
with open(str(outDataPath)) as f:
outData = json.load(f)
controlDozor = ControlDozor(inData={})
controlDozor.template = "mesh-test_1_%4d.cbf"
controlDozor.directory = UtilsTest.getTestImageDirPath().as_posix()
controlDozor.makePlot(dataCollectionId, outData, workingDirectory)
class ControlDozorUnitTest(unittest.TestCase):
def setUp(self):
self.dataPath = os.path.join(os.path.dirname(__file__), "data")
referenceDataPath = os.path.join(self.dataPath, "ControlDozor.json")
with open(referenceDataPath) as f:
self.inData = json.loads(f.read())
self.controlDozor = ControlDozor(inData=self.inData)
def testCreateDict(self):
dictImage = self.controlDozor.createImageDict(self.inData)
self.assertEqual(True, isinstance(dictImage, dict))
def testCreateListOfBatches(self):
self.assertEqual([[1], [2], [3], [4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 1))
self.assertEqual([[1, 2], [3, 4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 2))
self.assertEqual([[1, 2, 3], [4, 5]],
ControlDozor.createListOfBatches(range(1, 6), 3))
self.assertEqual([[1, 2, 3, 4], [5]],
ControlDozor.createListOfBatches(range(1, 6), 4))
self.assertEqual([[1, 2, 3, 4, 5]],
ControlDozor.createListOfBatches(range(1, 6), 5))
self.assertEqual(
[[1], [2], [4], [5], [6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 1),
)
self.assertEqual(
[[1, 2], [4, 5], [6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 2),
)
self.assertEqual(
[[1, 2], [4, 5, 6]],
ControlDozor.createListOfBatches(
list(range(4, 7)) + list(range(1, 3)), 3),
)
def runControlDozor(listSubWedge):
listControlDozor = []
listOutDataControlDozor = []
for subWedge in listSubWedge:
listSubWedgeImage = subWedge["image"]
for image in listSubWedgeImage:
# listImage.append(image['path'])
inDataControlDozor = {"image": [image["path"]]}
controlDozor = ControlDozor(
inData=inDataControlDozor,
workingDirectorySuffix=UtilsImage.getPrefixNumber(
image["path"]),
)
listControlDozor.append(controlDozor)
controlDozor.start()
for controlDozor in listControlDozor:
controlDozor.join()
if controlDozor.isSuccess():
listOutDataControlDozor.append(controlDozor.outData)
return listOutDataControlDozor
def setUp(self):
self.dataPath = os.path.join(os.path.dirname(__file__), "data")
referenceDataPath = os.path.join(self.dataPath, "ControlDozor.json")
with open(referenceDataPath) as f:
self.inData = json.loads(f.read())
self.controlDozor = ControlDozor(inData=self.inData)
def runDozorAndDistl(self, listOfBatches):
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and DistlSignalStrength (if required) in parallel
#
listDistlTask = []
listDozorTask = []
template4d = re.sub("#+", "{0:04d}", self.template)
for listOfImagesInBatch in listOfBatches:
listOfH5FilesInBatch = []
imageNo = listOfImagesInBatch[-1]
# Wait for last image
imagePath = self.directory / template4d.format(imageNo)
logger.debug("Waiting for path: {0}".format(imagePath))
self.waitForImagePath(
imagePath=imagePath,
batchSize=self.batchSize,
isFastMesh=self.isFastMesh,
minImageSize=self.minImageSize,
waitFileTimeOut=self.waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch,
)
logger.debug("Done waiting for path: {0}".format(imagePath))
if not self.isFailure():
# Determine start and end image no
batchStartNo = listOfImagesInBatch[0]
batchEndNo = listOfImagesInBatch[-1]
dozorTemplate = self.template
logger.debug("Before sleeping 2 s, start no = {0}".format(batchStartNo))
time.sleep(2)
logger.debug("After sleeping 2 s, start no = {0}".format(batchStartNo))
# Run Control Dozor
inDataControlDozor = {
"template": dozorTemplate,
"directory": self.directory,
"startNo": batchStartNo,
"endNo": batchEndNo,
"batchSize": self.batchSize,
"doSubmit": self.doSubmit,
"doDozorM": self.doDozorM,
"doIspybUpload": self.doIspybUpload,
"dataCollectionId": self.dataCollectionId
}
if self.beamline is not None:
inDataControlDozor["beamline"] = self.beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix="{0:04d}_{1:04d}".format(
batchStartNo, batchEndNo
),
)
controlDozor.start()
listDozorTask.append(
(controlDozor, inDataControlDozor, list(listOfImagesInBatch))
)
# Check if we should run distl.signalStrength
if self.doDistlSignalStrength:
for imageNo in listOfImagesInBatch:
imagePath = self.directory / template4d.format(imageNo)
inDataDistl = {"referenceImage": str(imagePath)}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=imageNo,
)
distlTask.start()
listDistlTask.append((imagePath, distlTask))
return listDozorTask, listDistlTask
def run(self, inData):
beamline = inData.get('beamline', None)
doSubmit = inData.get('doSubmit', False)
doDozorm = inData.get('doDozorm', False)
batchSize = inData.get('batchSize', 1)
doDistlSignalStrength = inData.get('doDistlSignalStrength', False)
doIndexing = inData.get('doIndexing', False)
if crystFelImportFailed:
doCrystfel = False
else:
doCrystfel = inData.get('doCrystfel', True)
isFastMesh = inData.get('fastMesh', False)
# Loop through all the incoming reference images
listImage = inData.get('image', [])
if len(listImage) == 0:
directory = pathlib.Path(inData['directory'])
template = inData['template']
startNo = inData['startNo']
endNo = inData['endNo']
listImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = directory / imageName
listImage.append(str(imagePath))
else:
firstImage = listImage[0]
lastImage = listImage[-1]
directory = pathlib.Path(firstImage).parent
template = UtilsImage.getTemplate(firstImage)
startNo = UtilsImage.getImageNumber(firstImage)
endNo = UtilsImage.getImageNumber(lastImage)
outData = dict()
listImageQualityIndicators = []
listcrystfel_output = []
inDataWaitFile = {}
listDistlTask = []
listDozorTask = []
listCrystFELTask = []
listOfImagesInBatch = []
listOfAllBatches = []
listOfAllH5Files = []
listControlDozorAllFile = []
indexBatch = 0
self.listH5FilePath = []
detectorType = None
# Configurations
minImageSize = UtilsConfig.get(
self, 'minImageSize', defaultValue=DEFAULT_MIN_IMAGE_SIZE)
waitFileTimeOut = UtilsConfig.get(
self, 'waitFileTimeOut', defaultValue=DEFAULT_WAIT_FILE_TIMEOUT)
# Process data in batches
for image in listImage:
listOfImagesInBatch.append(pathlib.Path(image))
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if UtilsImage.getSuffix(pathlib.Path(listImage[0])) == 'h5':
for image in listImage:
listOfAllH5Files.append(pathlib.Path(image))
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and Crystfel (if required) in parallel
#
# Check if we should run CrystFEL:
for listOfImagesInBatch in listOfAllBatches:
listOfH5FilesInBatch = []
for imagePath in listOfImagesInBatch:
# First wait for images
self.waitForImagePath(
imagePath=imagePath,
batchSize=batchSize,
isFastMesh=isFastMesh,
minImageSize=minImageSize,
waitFileTimeOut=waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch
)
if not self.isFailure():
# Determine start and end image no
pathToFirstImage = listOfImagesInBatch[0]
pathToLastImage = listOfImagesInBatch[-1]
batchStartNo = UtilsImage.getImageNumber(pathToFirstImage)
batchEndNo = UtilsImage.getImageNumber(pathToLastImage)
# Run Control Dozor
inDataControlDozor = {
'template': template,
'directory': directory,
'startNo': batchStartNo,
'endNo': batchEndNo,
'batchSize': batchSize,
'doSubmit': doSubmit,
'doDozorm': doDozorm
}
if beamline is not None:
inDataControlDozor["beamline"] = beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix='{0:04d}_{1:04d}'.format(batchStartNo, batchEndNo))
controlDozor.start()
listDozorTask.append((controlDozor, inDataControlDozor,
list(listOfImagesInBatch), listOfH5FilesInBatch))
# Check if we should run distl.signalStrength
if doDistlSignalStrength:
for image in listOfImagesInBatch:
inDataDistl = {
'referenceImage': str(image)
}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=UtilsImage.getImageNumber(image))
distlTask.start()
listDistlTask.append((image, distlTask))
if not self.isFailure():
# listIndexing = []
listDistlResult = []
# Synchronize all image quality indicator plugins and upload to ISPyB
for (image, distlTask) in listDistlTask:
imageQualityIndicators = {}
if distlTask is not None:
distlTask.join()
if distlTask.isSuccess():
outDataDistl = distlTask.outData
if outDataDistl is not None:
imageQualityIndicators = outDataDistl['imageQualityIndicators']
imageQualityIndicators['image'] = str(image)
listDistlResult.append(imageQualityIndicators)
# self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (controlDozor, inDataControlDozor, listBatch, listH5FilePath) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData['imageQualityIndicators']) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0].name
lastImage = listBatch[-1].name
logger.warning("No dozor results! Re-executing Dozor for" +
" images {0} to {1}".format(firstImage, lastImage))
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(controlDozor.outData['imageQualityIndicators'])
if detectorType is None:
detectorType = controlDozor.outData['detectorType']
if doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor['image'] == distlResult['image']:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if doDozorm:
listControlDozorAllFile.append(controlDozor.outData['dozorAllFile'])
if not self.isFailure() and doCrystfel:
# Select only the strongest images to be run by CrystFEL
listIndicatorsSorted = sorted(listImageQualityIndicators, key=lambda k: k['dozorScore'])[::-1]
listForCrystFEL = [k['image'] for k in listIndicatorsSorted[0:min(200, len(listIndicatorsSorted))]]
batchSize = 20
if len(listForCrystFEL) % batchSize == 0:
file_chunk = int(len(listForCrystFEL) / batchSize)
else:
file_chunk = int(len(listForCrystFEL) / batchSize) + 1
for jj in range(file_chunk):
start = batchSize * jj
stop = batchSize * (jj + 1)
try:
images = listForCrystFEL[start:stop]
except IndexError:
stop = start + (len(listForCrystFEL) - stop)
images = listForCrystFEL[start:stop]
inDataCrystFEL = {
'doCBFtoH5': False,
'cbfFileInfo': {
'directory': directory,
'template': template,
'batchSize': batchSize,
'listofImages': images
},
'doSubmit': doSubmit
}
crystfel = ExeCrystFEL(inData=inDataCrystFEL)
crystfel.start()
listCrystFELTask.append(crystfel)
masterstream = str(self.getWorkingDirectory() / 'alltogether.stream')
if not self.isFailure():
for crystfel in listCrystFELTask:
crystfel.join()
if crystfel.isSuccess():
catcommand = "cat %s >> %s" % (crystfel.outData['streamfile'], masterstream)
AutoCrystFEL.run_as_command(catcommand)
if not self.isFailure() and os.path.exists(masterstream):
crystfel_outdata = AutoCrystFEL.report_stats(masterstream)
crystfel_outdata['number of DozorHits'] = len(listForCrystFEL)
AutoCrystFEL.write_cell_file(crystfel_outdata)
listcrystfel_output.append(crystfel_outdata)
else:
logger.error("CrystFEL did not run properly")
# Assemble all controlDozorAllFiles into one
if doDozorm:
imageQualityIndicatorsDozorAllFile = str(self.getWorkingDirectory() / "dozor_all")
os.system('touch {0}'.format(imageQualityIndicatorsDozorAllFile))
for controlDozorAllFile in listControlDozorAllFile:
command = 'cat ' + controlDozorAllFile + ' >> ' + imageQualityIndicatorsDozorAllFile
os.system(command)
outData["dozorAllFile"] = imageQualityIndicatorsDozorAllFile
outData['imageQualityIndicators'] = listImageQualityIndicators
outData['crystfel_results'] = listcrystfel_output
return outData