def createBatchList(self, inData):
listOfBatches = []
listOfImagesInBatch = []
if len(self.listImage) == 0:
self.directory = pathlib.Path(inData["directory"])
self.template = inData["template"]
startNo = inData["startNo"]
endNo = inData["endNo"]
for index in range(startNo, endNo + 1):
listOfImagesInBatch.append(index)
if len(listOfImagesInBatch) == self.batchSize or index == 9999:
listOfBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
else:
firstImage = pathlib.Path(self.listImage[0])
self.directory = firstImage.parent
self.template = UtilsImage.getTemplate(firstImage)
for image in self.listImage:
imageNo = UtilsImage.getImageNumber(image)
listOfImagesInBatch.append(imageNo)
if len(listOfImagesInBatch) == self.batchSize:
listOfBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfBatches.append(listOfImagesInBatch)
return listOfBatches
def test_getTemplateQuestionMark(self):
template = UtilsImage.getTemplate(self.imageFileName, symbol="?")
templateReference = "ref-testscale_1_????.img"
self.assertEqual(templateReference, template)
def test_getTemplateHash2(self):
template = UtilsImage.getTemplate(self.imageFileName2)
templateReference = "ref-testscale_1_#####.img"
self.assertEqual(templateReference, template)
def generateMOSFLMInData(cls, inData):
if "imagePath" in inData:
inDataReadImageHeader = {"imagePath": inData["imagePath"]}
readImageHeader = ReadImageHeader(inData=inDataReadImageHeader)
readImageHeader.execute()
listSubWedges = readImageHeader.outData["subWedge"]
elif "subWedge" in inData:
listSubWedges = inData["subWedge"]
else:
# Here we assume that we have inData in MOSFLM format...
return inData
firstSubWedge = listSubWedges[0]
experimentalCondition = firstSubWedge["experimentalCondition"]
detector = experimentalCondition["detector"]
beam = experimentalCondition["beam"]
goniostat = experimentalCondition["goniostat"]
detectorType = detector["type"]
if detectorType.startswith("pilatus"):
mosflmDetector = "PILATUS"
elif detectorType.startswith("eiger"):
mosflmDetector = "EIGER"
else:
raise RuntimeError("Unknown detecor type for MOSFLM: " +
detectorType)
listImage = firstSubWedge["image"]
firstImage = listImage[0]
firstPath = firstImage["path"]
mosflmInData = {
# The MOSFLM beam position convention is opposite to the XDS / CBF/ Dozor convention
"beam": {
"x": detector["beamPositionY"],
"y": detector["beamPositionX"]
},
"detector": {
"numberPixelX": detector["numberPixelX"],
"numberPixelY": detector["numberPixelY"],
"pixelSizeX": detector["pixelSizeX"],
"pixelSizeY": detector["pixelSizeY"],
"type": mosflmDetector
},
"directory": os.path.dirname(firstPath),
"distance": detector["distance"],
"template": UtilsImage.getTemplate(firstPath),
"wavelength": beam["wavelength"]
}
listMosflmImage = []
for subWedge in listSubWedges:
goniostat = subWedge["experimentalCondition"]["goniostat"]
rotationAxisStart = goniostat['rotationAxisStart']
oscillationWidth = goniostat['oscillationWidth']
imageIndex = 0
for image in subWedge["image"]:
number = UtilsImage.getImageNumber(image["path"])
rotationAxisStart = rotationAxisStart + oscillationWidth * imageIndex
rotationAxisEnd = rotationAxisStart + oscillationWidth
mosflmImage = {
"number": number,
"rotationAxisStart": rotationAxisStart,
"rotationAxisEnd": rotationAxisEnd
}
listMosflmImage.append(mosflmImage)
mosflmInData["image"] = listMosflmImage
return mosflmInData
def run(self, inData):
beamline = inData.get('beamline', None)
doSubmit = inData.get('doSubmit', False)
doDozorm = inData.get('doDozorm', False)
batchSize = inData.get('batchSize', 1)
doDistlSignalStrength = inData.get('doDistlSignalStrength', False)
doIndexing = inData.get('doIndexing', False)
if crystFelImportFailed:
doCrystfel = False
else:
doCrystfel = inData.get('doCrystfel', True)
isFastMesh = inData.get('fastMesh', False)
# Loop through all the incoming reference images
listImage = inData.get('image', [])
if len(listImage) == 0:
directory = pathlib.Path(inData['directory'])
template = inData['template']
startNo = inData['startNo']
endNo = inData['endNo']
listImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = directory / imageName
listImage.append(str(imagePath))
else:
firstImage = listImage[0]
lastImage = listImage[-1]
directory = pathlib.Path(firstImage).parent
template = UtilsImage.getTemplate(firstImage)
startNo = UtilsImage.getImageNumber(firstImage)
endNo = UtilsImage.getImageNumber(lastImage)
outData = dict()
listImageQualityIndicators = []
listcrystfel_output = []
inDataWaitFile = {}
listDistlTask = []
listDozorTask = []
listCrystFELTask = []
listOfImagesInBatch = []
listOfAllBatches = []
listOfAllH5Files = []
listControlDozorAllFile = []
indexBatch = 0
self.listH5FilePath = []
detectorType = None
# Configurations
minImageSize = UtilsConfig.get(
self, 'minImageSize', defaultValue=DEFAULT_MIN_IMAGE_SIZE)
waitFileTimeOut = UtilsConfig.get(
self, 'waitFileTimeOut', defaultValue=DEFAULT_WAIT_FILE_TIMEOUT)
# Process data in batches
for image in listImage:
listOfImagesInBatch.append(pathlib.Path(image))
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if UtilsImage.getSuffix(pathlib.Path(listImage[0])) == 'h5':
for image in listImage:
listOfAllH5Files.append(pathlib.Path(image))
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and Crystfel (if required) in parallel
#
# Check if we should run CrystFEL:
for listOfImagesInBatch in listOfAllBatches:
listOfH5FilesInBatch = []
for imagePath in listOfImagesInBatch:
# First wait for images
self.waitForImagePath(
imagePath=imagePath,
batchSize=batchSize,
isFastMesh=isFastMesh,
minImageSize=minImageSize,
waitFileTimeOut=waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch
)
if not self.isFailure():
# Determine start and end image no
pathToFirstImage = listOfImagesInBatch[0]
pathToLastImage = listOfImagesInBatch[-1]
batchStartNo = UtilsImage.getImageNumber(pathToFirstImage)
batchEndNo = UtilsImage.getImageNumber(pathToLastImage)
# Run Control Dozor
inDataControlDozor = {
'template': template,
'directory': directory,
'startNo': batchStartNo,
'endNo': batchEndNo,
'batchSize': batchSize,
'doSubmit': doSubmit,
'doDozorm': doDozorm
}
if beamline is not None:
inDataControlDozor["beamline"] = beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix='{0:04d}_{1:04d}'.format(batchStartNo, batchEndNo))
controlDozor.start()
listDozorTask.append((controlDozor, inDataControlDozor,
list(listOfImagesInBatch), listOfH5FilesInBatch))
# Check if we should run distl.signalStrength
if doDistlSignalStrength:
for image in listOfImagesInBatch:
inDataDistl = {
'referenceImage': str(image)
}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=UtilsImage.getImageNumber(image))
distlTask.start()
listDistlTask.append((image, distlTask))
if not self.isFailure():
# listIndexing = []
listDistlResult = []
# Synchronize all image quality indicator plugins and upload to ISPyB
for (image, distlTask) in listDistlTask:
imageQualityIndicators = {}
if distlTask is not None:
distlTask.join()
if distlTask.isSuccess():
outDataDistl = distlTask.outData
if outDataDistl is not None:
imageQualityIndicators = outDataDistl['imageQualityIndicators']
imageQualityIndicators['image'] = str(image)
listDistlResult.append(imageQualityIndicators)
# self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (controlDozor, inDataControlDozor, listBatch, listH5FilePath) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData['imageQualityIndicators']) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0].name
lastImage = listBatch[-1].name
logger.warning("No dozor results! Re-executing Dozor for" +
" images {0} to {1}".format(firstImage, lastImage))
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(controlDozor.outData['imageQualityIndicators'])
if detectorType is None:
detectorType = controlDozor.outData['detectorType']
if doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor['image'] == distlResult['image']:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if doDozorm:
listControlDozorAllFile.append(controlDozor.outData['dozorAllFile'])
if not self.isFailure() and doCrystfel:
# Select only the strongest images to be run by CrystFEL
listIndicatorsSorted = sorted(listImageQualityIndicators, key=lambda k: k['dozorScore'])[::-1]
listForCrystFEL = [k['image'] for k in listIndicatorsSorted[0:min(200, len(listIndicatorsSorted))]]
batchSize = 20
if len(listForCrystFEL) % batchSize == 0:
file_chunk = int(len(listForCrystFEL) / batchSize)
else:
file_chunk = int(len(listForCrystFEL) / batchSize) + 1
for jj in range(file_chunk):
start = batchSize * jj
stop = batchSize * (jj + 1)
try:
images = listForCrystFEL[start:stop]
except IndexError:
stop = start + (len(listForCrystFEL) - stop)
images = listForCrystFEL[start:stop]
inDataCrystFEL = {
'doCBFtoH5': False,
'cbfFileInfo': {
'directory': directory,
'template': template,
'batchSize': batchSize,
'listofImages': images
},
'doSubmit': doSubmit
}
crystfel = ExeCrystFEL(inData=inDataCrystFEL)
crystfel.start()
listCrystFELTask.append(crystfel)
masterstream = str(self.getWorkingDirectory() / 'alltogether.stream')
if not self.isFailure():
for crystfel in listCrystFELTask:
crystfel.join()
if crystfel.isSuccess():
catcommand = "cat %s >> %s" % (crystfel.outData['streamfile'], masterstream)
AutoCrystFEL.run_as_command(catcommand)
if not self.isFailure() and os.path.exists(masterstream):
crystfel_outdata = AutoCrystFEL.report_stats(masterstream)
crystfel_outdata['number of DozorHits'] = len(listForCrystFEL)
AutoCrystFEL.write_cell_file(crystfel_outdata)
listcrystfel_output.append(crystfel_outdata)
else:
logger.error("CrystFEL did not run properly")
# Assemble all controlDozorAllFiles into one
if doDozorm:
imageQualityIndicatorsDozorAllFile = str(self.getWorkingDirectory() / "dozor_all")
os.system('touch {0}'.format(imageQualityIndicatorsDozorAllFile))
for controlDozorAllFile in listControlDozorAllFile:
command = 'cat ' + controlDozorAllFile + ' >> ' + imageQualityIndicatorsDozorAllFile
os.system(command)
outData["dozorAllFile"] = imageQualityIndicatorsDozorAllFile
outData['imageQualityIndicators'] = listImageQualityIndicators
outData['crystfel_results'] = listcrystfel_output
return outData