def generateImageLinks(in_data, working_directory=None):
first_sub_wedge = in_data["subWedge"][0]
first_image_path = first_sub_wedge["image"][0]["path"]
prefix = UtilsImage.getPrefix(first_image_path)
suffix = UtilsImage.getSuffix(first_image_path)
if suffix == "h5":
lowest_xds_image_number = 1
highest_xds_image_number = 1
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
first_image_path, hasOverlap=True, isFastMesh=False)
h5MasterFile = os.path.basename((str(h5MasterFilePath)))
h5DataFile = os.path.basename((str(h5DataFilePath)))
list_image_link = [
[str(h5MasterFilePath), h5MasterFile],
[str(h5DataFilePath), h5DataFile],
]
if working_directory is not None:
os.symlink(str(h5MasterFilePath), h5MasterFile)
os.symlink(str(h5DataFilePath), h5DataFile)
template = h5MasterFile.replace("master", "??????")
lowest_xds_image_number = None
highest_xds_image_number = None
for subwedge in in_data["subWedge"]:
image_list = subwedge["image"]
for image_dict in image_list:
image_path = image_dict["path"]
image_number = UtilsImage.getImageNumber(image_path)
if lowest_xds_image_number is None or lowest_xds_image_number > image_number:
lowest_xds_image_number = image_number
if highest_xds_image_number is None or highest_xds_image_number < image_number:
highest_xds_image_number = image_number
else:
template = "%s_xdslink_?????.%s" % (prefix, suffix)
xds_lowest_image_number_global = 1
# First we have to find the smallest goniostat rotation axis start:
oscillation_start_min = 0
# Loop through the list of sub wedges
list_of_list = []
lowest_xds_image_number = None
highest_xds_image_number = None
list_spot_range = []
for sub_wedge in in_data["subWedge"]:
list_image_link = []
image_list = sub_wedge["image"]
goniostat = sub_wedge["experimentalCondition"]["goniostat"]
oscillation_start = goniostat["rotationAxisStart"]
oscillation_range = goniostat["oscillationWidth"]
# First find the lowest and highest image numbers
lowest_image_number = None
for image in image_list:
image_number = image["number"]
if lowest_image_number is None or image_number < lowest_image_number:
lowest_image_number = image_number
# Loop through the list of images
spot_range_min = None
spot_range_max = None
for image in image_list:
image_number = image["number"]
image_oscillation_start = (
oscillation_start +
(image_number - lowest_image_number) *
oscillation_range)
# if xdsLowestImageNumberGlobal is None:
# xdsLowestImageNumberGlobal = 1 + int((imageOscillationStart - oscillationStartMin) / oscillationRange)
xds_image_number = xds_lowest_image_number_global + int(
(image_oscillation_start - oscillation_start_min) /
oscillation_range + 0.5)
print(
xds_image_number,
image_oscillation_start,
oscillation_start_min,
oscillation_range,
)
source_path = image["path"]
target = "%s_xdslink_%05d.%s" % (prefix, xds_image_number,
suffix)
print([source_path, target])
list_image_link.append([source_path, target])
if working_directory is not None and not os.path.exists(
target):
os.symlink(source_path, target)
if (lowest_xds_image_number is None
or lowest_xds_image_number > xds_image_number):
lowest_xds_image_number = xds_image_number
if (highest_xds_image_number is None
or highest_xds_image_number < xds_image_number):
highest_xds_image_number = xds_image_number
if spot_range_min is None or spot_range_min > xds_image_number:
spot_range_min = xds_image_number
if spot_range_max is None or spot_range_max < xds_image_number:
spot_range_max = xds_image_number
list_spot_range.append([spot_range_min, spot_range_max])
list_of_list.append(list_image_link)
previous_exclude_data_range_max = 1
list_exclude_data_range = []
for spot_range_min, spot_range_max in list_spot_range:
if spot_range_min > previous_exclude_data_range_max + 1:
list_exclude_data_range.append(
[previous_exclude_data_range_max, spot_range_min - 1])
previous_exclude_data_range_max = spot_range_max + 1
dictImageLinks = {
"imageLink": list_of_list,
"spotRange": list_spot_range,
"dataRange": [lowest_xds_image_number, highest_xds_image_number],
"excludeDataRange": list_exclude_data_range,
"template": template,
}
return dictImageLinks
def exeIndexing(self, inData):
doCBFtoH5 = inData.get('doCBFtoH5', False)
in_for_crystfel = dict()
if 'listH5FilePath' in inData.keys():
tmp = UtilsImage.getPrefix(inData['listH5FilePath'][0])
FirstImage = tmp.replace('data', 'master.h5')
Image = Im(FirstImage)
in_for_crystfel['detectorType'] = Image.imobject.headers['detector_name'][0] + \
Image.imobject.headers['detector_name'][1]
in_for_crystfel['prefix'] = tmp.strip('data')
in_for_crystfel['suffix'] = UtilsImage.getSuffix(inData['listH5FilePath'][0])
in_for_crystfel['image_directory'] = str(pathlib.Path(inData['listH5FilePath'][0]).parent)
in_for_crystfel['maxchunksize'] = 10
elif 'cbfFileInfo' in inData.keys():
in_for_crystfel['maxchunksize'] = inData['cbfFileInfo'].get('batchSize', 10)
in_for_crystfel['listofImages'] = inData['cbfFileInfo'].get('listofImages', [])
in_for_crystfel['image_directory'] = inData['cbfFileInfo']['directory']
in_for_crystfel['prefix'] = inData['cbfFileInfo']['template'].strip('####.cbf')
in_for_crystfel['suffix'] = UtilsImage.getSuffix(inData['cbfFileInfo']['template'])
if len(in_for_crystfel['listofImages']) == 0:
in_for_crystfel['ImageRange'] = (inData['cbfFileInfo']['startNo'], inData['cbfFileInfo']['endNo'])
FirstImage = os.path.join(inData['cbfFileInfo']['directory'], inData['cbfFileInfo']['template'].
replace('####', '0001'))
else:
FirstImage = in_for_crystfel['listofImages'][0]
Image = Im(FirstImage)
in_for_crystfel['detectorType'] = Image.imobject.headers['detector_name'][0] + \
Image.imobject.headers['detector_name'][1]
else:
logger.error("input json must have either listH5FilePath or cbfFileInfo")
if doCBFtoH5:
cxi_all = list(self.getWorkingDirectory().glob('dozor*cxi'))
current = len(cxi_all) - 1
in_for_crystfel['image_directory'] = self.getWorkingDirectory()
in_for_crystfel['prefix'] = 'dozor_%d.' % current
in_for_crystfel['suffix'] = 'cxi'
in_for_crystfel['peak_search'] = 'cxi'
in_for_crystfel['peak_info'] = '/data/peakinfo'
in_for_crystfel['maxchunksize'] = 10
crysttask = run_crystfel.AutoCrystFEL(in_for_crystfel)
outstream = None
try:
jsonschema.validate(instance=crysttask.jshandle, schema=crysttask.getInDataSchema())
crysttask.datafinder()
crysttask.makeOutputDirectory()
kk = {}
if crysttask.jshandle['suffix'] == 'cxi':
kk['cxi'] = """dim0 = %\ndim1 = ss\ndim2 = fs\ndata = /data/data\n"""
geomfile = crysttask.make_geometry_file(**kk)
else:
geomfile = crysttask.make_geometry_file(**kk)
crysttask.make_list_events(str(geomfile))
infile = str(crysttask.getOutputDirectory() / 'input.lst')
outname = datetime.now().strftime('%H-%M-%S.stream')
outstream = str(crysttask.getOutputDirectory() / outname)
ofh = open(infile, 'w')
for fname in crysttask.filelist:
ofh.write(fname)
ofh.write('\n')
ofh.close()
crystfel_cmd = crysttask.indexamajig_cmd(infile, outstream, geomfile)
self.runCommandLine(crystfel_cmd, doSubmit=inData.get('doSubmit', True))
except Exception as err:
self.setFailure()
logger.error(err)
return outstream, crysttask
def generateImageLinks(inData, workingDirectory=None):
listImageLink = []
firstSubWedge = inData["subWedge"][0]
firstImagePath = firstSubWedge["image"][0]["path"]
prefix = UtilsImage.getPrefix(firstImagePath)
suffix = UtilsImage.getSuffix(firstImagePath)
template = "%s_xdslink_?????.%s" % (prefix, suffix)
xdsLowestImageNumberGlobal = 1
# First we have to find the smallest goniostat rotation axis start:
oscillationStartMin = 0
# for subWedge in inData["subWedge"]:
# goniostat = subWedge["experimentalCondition"]["goniostat"]
# oscillationStart = goniostat["rotationAxisStart"]
# if oscillationStartMin is None or \
# oscillationStartMin > oscillationStart:
# oscillationStartMin = oscillationStart
# Loop through the list of sub wedges
for subWedge in inData["subWedge"]:
imageList = subWedge["image"]
xsDataGoniostat = subWedge["experimentalCondition"]["goniostat"]
oscillationStart = xsDataGoniostat["rotationAxisStart"]
oscillationRange = xsDataGoniostat["oscillationWidth"]
# First find the lowest and highest image numbers
lowestImageNumber = None
for dictImage in imageList:
imageNumber = dictImage["number"]
if lowestImageNumber is None or imageNumber < lowestImageNumber:
lowestImageNumber = imageNumber
# Loop through the list of images
lowestXDSImageNumber = None
highestXDSImageNumber = None
for dictImage in imageList:
imageNumber = dictImage["number"]
imageOscillationStart = \
oscillationStart + (imageNumber - lowestImageNumber) * oscillationRange
# if xdsLowestImageNumberGlobal is None:
# xdsLowestImageNumberGlobal = 1 + int((imageOscillationStart - oscillationStartMin) / oscillationRange)
xdsImageNumber = xdsLowestImageNumberGlobal + \
int((imageOscillationStart - oscillationStartMin) / oscillationRange)
print(xdsImageNumber, imageOscillationStart,
oscillationStartMin, oscillationRange)
sourcePath = dictImage["path"]
target = "%s_xdslink_%05d.%s" % (prefix, xdsImageNumber,
suffix)
print([sourcePath, target])
listImageLink.append([sourcePath, target])
if workingDirectory is not None:
os.symlink(sourcePath, target)
if lowestXDSImageNumber is None or \
lowestXDSImageNumber > xdsImageNumber:
lowestXDSImageNumber = xdsImageNumber
if highestXDSImageNumber is None or \
highestXDSImageNumber < xdsImageNumber:
highestXDSImageNumber = xdsImageNumber
dictImageLinks = {
"imageLink": listImageLink,
"dataRange": [lowestXDSImageNumber, highestXDSImageNumber],
"template": template
}
return dictImageLinks
def run(self, inData):
beamline = inData.get('beamline', None)
doSubmit = inData.get('doSubmit', False)
doDozorm = inData.get('doDozorm', False)
batchSize = inData.get('batchSize', 1)
doDistlSignalStrength = inData.get('doDistlSignalStrength', False)
doIndexing = inData.get('doIndexing', False)
if crystFelImportFailed:
doCrystfel = False
else:
doCrystfel = inData.get('doCrystfel', True)
isFastMesh = inData.get('fastMesh', False)
# Loop through all the incoming reference images
listImage = inData.get('image', [])
if len(listImage) == 0:
directory = pathlib.Path(inData['directory'])
template = inData['template']
startNo = inData['startNo']
endNo = inData['endNo']
listImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = directory / imageName
listImage.append(str(imagePath))
else:
firstImage = listImage[0]
lastImage = listImage[-1]
directory = pathlib.Path(firstImage).parent
template = UtilsImage.getTemplate(firstImage)
startNo = UtilsImage.getImageNumber(firstImage)
endNo = UtilsImage.getImageNumber(lastImage)
outData = dict()
listImageQualityIndicators = []
listcrystfel_output = []
inDataWaitFile = {}
listDistlTask = []
listDozorTask = []
listCrystFELTask = []
listOfImagesInBatch = []
listOfAllBatches = []
listOfAllH5Files = []
listControlDozorAllFile = []
indexBatch = 0
self.listH5FilePath = []
detectorType = None
# Configurations
minImageSize = UtilsConfig.get(
self, 'minImageSize', defaultValue=DEFAULT_MIN_IMAGE_SIZE)
waitFileTimeOut = UtilsConfig.get(
self, 'waitFileTimeOut', defaultValue=DEFAULT_WAIT_FILE_TIMEOUT)
# Process data in batches
for image in listImage:
listOfImagesInBatch.append(pathlib.Path(image))
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if UtilsImage.getSuffix(pathlib.Path(listImage[0])) == 'h5':
for image in listImage:
listOfAllH5Files.append(pathlib.Path(image))
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and Crystfel (if required) in parallel
#
# Check if we should run CrystFEL:
for listOfImagesInBatch in listOfAllBatches:
listOfH5FilesInBatch = []
for imagePath in listOfImagesInBatch:
# First wait for images
self.waitForImagePath(
imagePath=imagePath,
batchSize=batchSize,
isFastMesh=isFastMesh,
minImageSize=minImageSize,
waitFileTimeOut=waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch
)
if not self.isFailure():
# Determine start and end image no
pathToFirstImage = listOfImagesInBatch[0]
pathToLastImage = listOfImagesInBatch[-1]
batchStartNo = UtilsImage.getImageNumber(pathToFirstImage)
batchEndNo = UtilsImage.getImageNumber(pathToLastImage)
# Run Control Dozor
inDataControlDozor = {
'template': template,
'directory': directory,
'startNo': batchStartNo,
'endNo': batchEndNo,
'batchSize': batchSize,
'doSubmit': doSubmit,
'doDozorm': doDozorm
}
if beamline is not None:
inDataControlDozor["beamline"] = beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix='{0:04d}_{1:04d}'.format(batchStartNo, batchEndNo))
controlDozor.start()
listDozorTask.append((controlDozor, inDataControlDozor,
list(listOfImagesInBatch), listOfH5FilesInBatch))
# Check if we should run distl.signalStrength
if doDistlSignalStrength:
for image in listOfImagesInBatch:
inDataDistl = {
'referenceImage': str(image)
}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=UtilsImage.getImageNumber(image))
distlTask.start()
listDistlTask.append((image, distlTask))
if not self.isFailure():
# listIndexing = []
listDistlResult = []
# Synchronize all image quality indicator plugins and upload to ISPyB
for (image, distlTask) in listDistlTask:
imageQualityIndicators = {}
if distlTask is not None:
distlTask.join()
if distlTask.isSuccess():
outDataDistl = distlTask.outData
if outDataDistl is not None:
imageQualityIndicators = outDataDistl['imageQualityIndicators']
imageQualityIndicators['image'] = str(image)
listDistlResult.append(imageQualityIndicators)
# self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (controlDozor, inDataControlDozor, listBatch, listH5FilePath) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData['imageQualityIndicators']) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0].name
lastImage = listBatch[-1].name
logger.warning("No dozor results! Re-executing Dozor for" +
" images {0} to {1}".format(firstImage, lastImage))
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(controlDozor.outData['imageQualityIndicators'])
if detectorType is None:
detectorType = controlDozor.outData['detectorType']
if doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor['image'] == distlResult['image']:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if doDozorm:
listControlDozorAllFile.append(controlDozor.outData['dozorAllFile'])
if not self.isFailure() and doCrystfel:
# Select only the strongest images to be run by CrystFEL
listIndicatorsSorted = sorted(listImageQualityIndicators, key=lambda k: k['dozorScore'])[::-1]
listForCrystFEL = [k['image'] for k in listIndicatorsSorted[0:min(200, len(listIndicatorsSorted))]]
batchSize = 20
if len(listForCrystFEL) % batchSize == 0:
file_chunk = int(len(listForCrystFEL) / batchSize)
else:
file_chunk = int(len(listForCrystFEL) / batchSize) + 1
for jj in range(file_chunk):
start = batchSize * jj
stop = batchSize * (jj + 1)
try:
images = listForCrystFEL[start:stop]
except IndexError:
stop = start + (len(listForCrystFEL) - stop)
images = listForCrystFEL[start:stop]
inDataCrystFEL = {
'doCBFtoH5': False,
'cbfFileInfo': {
'directory': directory,
'template': template,
'batchSize': batchSize,
'listofImages': images
},
'doSubmit': doSubmit
}
crystfel = ExeCrystFEL(inData=inDataCrystFEL)
crystfel.start()
listCrystFELTask.append(crystfel)
masterstream = str(self.getWorkingDirectory() / 'alltogether.stream')
if not self.isFailure():
for crystfel in listCrystFELTask:
crystfel.join()
if crystfel.isSuccess():
catcommand = "cat %s >> %s" % (crystfel.outData['streamfile'], masterstream)
AutoCrystFEL.run_as_command(catcommand)
if not self.isFailure() and os.path.exists(masterstream):
crystfel_outdata = AutoCrystFEL.report_stats(masterstream)
crystfel_outdata['number of DozorHits'] = len(listForCrystFEL)
AutoCrystFEL.write_cell_file(crystfel_outdata)
listcrystfel_output.append(crystfel_outdata)
else:
logger.error("CrystFEL did not run properly")
# Assemble all controlDozorAllFiles into one
if doDozorm:
imageQualityIndicatorsDozorAllFile = str(self.getWorkingDirectory() / "dozor_all")
os.system('touch {0}'.format(imageQualityIndicatorsDozorAllFile))
for controlDozorAllFile in listControlDozorAllFile:
command = 'cat ' + controlDozorAllFile + ' >> ' + imageQualityIndicatorsDozorAllFile
os.system(command)
outData["dozorAllFile"] = imageQualityIndicatorsDozorAllFile
outData['imageQualityIndicators'] = listImageQualityIndicators
outData['crystfel_results'] = listcrystfel_output
return outData