def run(self, inData):
outData = {}
# First get the list of subWedges
if "subWedge" in inData:
list_sub_wedge = inData["subWedge"]
directory_prefix = ""
else:
first_image = inData["imagePath"][0]
last_image = inData["imagePath"][-1]
prefix = UtilsImage.getPrefix(first_image)
first_image_number = UtilsImage.getImageNumber((first_image))
last_image_number = UtilsImage.getImageNumber((last_image))
directory_prefix = "{0}_{1}_{2}".format(
prefix, first_image_number, last_image_number
)
list_sub_wedge = self.getListSubWedge(inData)
indata_xds_integration = {
"subWedge": list_sub_wedge
}
xds_integration = XDSIndexAndIntegration(
inData=indata_xds_integration,
workingDirectorySuffix=directory_prefix,
)
xds_integration.execute()
outData = xds_integration.outData
return outData
def createBatchList(self, inData):
listOfBatches = []
listOfImagesInBatch = []
if len(self.listImage) == 0:
self.directory = pathlib.Path(inData["directory"])
self.template = inData["template"]
startNo = inData["startNo"]
endNo = inData["endNo"]
for index in range(startNo, endNo + 1):
listOfImagesInBatch.append(index)
if len(listOfImagesInBatch) == self.batchSize or index == 9999:
listOfBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
else:
firstImage = pathlib.Path(self.listImage[0])
self.directory = firstImage.parent
self.template = UtilsImage.getTemplate(firstImage)
for image in self.listImage:
imageNo = UtilsImage.getImageNumber(image)
listOfImagesInBatch.append(imageNo)
if len(listOfImagesInBatch) == self.batchSize:
listOfBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfBatches.append(listOfImagesInBatch)
return listOfBatches
def getH5FilePath(cls, filePath, batchSize=1, isFastMesh=False):
imageNumber = UtilsImage.getImageNumber(filePath)
prefix = UtilsImage.getPrefix(filePath)
if isFastMesh:
h5ImageNumber = int((imageNumber - 1) / 100) + 1
h5FileNumber = 1
else:
h5ImageNumber = 1
h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1
h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format(
prefix=prefix, h5FileNumber=h5FileNumber)
h5MasterFilePath = filePath.parent / h5MasterFileName
h5DataFileName = \
"{prefix}_{h5FileNumber}_data_{h5ImageNumber:06d}.h5".format(
prefix=prefix, h5FileNumber=h5FileNumber, h5ImageNumber=h5ImageNumber)
h5DataFilePath = filePath.parent / h5DataFileName
return h5MasterFilePath, h5DataFilePath, h5FileNumber
def findDataCollectionFromFileLocationAndFileName(imagePath, client=None):
logger = UtilsLogging.getLogger()
dataCollectionWS3VO = None
noTrials = 10
fileLocation = os.path.dirname(imagePath)
fileName = os.path.basename(imagePath)
if fileName.endswith(".h5"):
prefix = UtilsImage.getPrefix(fileName)
imageNumber = UtilsImage.getImageNumber(fileName)
fileName = "{0}_{1:04d}.h5".format(prefix, imageNumber)
try:
if client is None:
client = getCollectionWebService()
if client is None:
logger.error(
"No web service client available, cannot contact findDataCollectionFromFileLocationAndFileName web service."
)
elif fileLocation is None:
logger.error(
"No fileLocation given, cannot contact findDataCollectionFromFileLocationAndFileName web service."
)
elif fileName is None:
logger.error(
"No fileName given, cannot contact findDataCollectionFromFileLocationAndFileName web service."
)
else:
dataCollectionWS3VO = client.service.findDataCollectionFromFileLocationAndFileName(
fileLocation, fileName)
except Exception as e:
logger.error(
"ISPyB error for findDataCollectionFromFileLocationAndFileName: {0}, {1} trials left"
.format(e, noTrials))
raise e
if dataCollectionWS3VO is None:
time.sleep(1)
if noTrials == 0:
raise RuntimeError(
"ISPyB error for findDataCollectionFromFileLocationAndFileName - no data collections found for path {0}"
.format(imagePath))
else:
logger.warning(
"Cannot find {0} in ISPyB - retrying, {1} trials left".format(
imagePath, noTrials))
return dataCollectionWS3VO
def createCBFHeaderData(cls, imagePath):
# Waiting for file
timedOut, finalSize = UtilsPath.waitForFile(imagePath,
expectedSize=100000,
timeOut=DEFAULT_TIME_OUT)
if timedOut:
errorMessage = "Timeout when waiting for image %s" % imagePath
logger.error(errorMessage)
raise BaseException(errorMessage)
dictHeader = cls.readCBFHeader(imagePath)
detector = dictHeader['Detector:']
if 'PILATUS 3M' in detector or 'PILATUS3 2M' in detector or \
'PILATUS 2M' in detector or 'PILATUS2 3M' in detector:
detectorName = 'PILATUS2 3M'
detectorType = 'pilatus2m'
numberPixelX = 1475
numberPixelY = 1679
elif 'PILATUS 6M' in detector or 'PILATUS3 6M' in detector:
detectorName = 'PILATUS2 6M'
detectorType = 'pilatus6m'
numberPixelX = 2463
numberPixelY = 2527
else:
raise RuntimeError(
'{0} cannot read image header from images with dector type {1}'
.format(cls.__class__.__name__, detector))
experimentalCondition = {}
detector = {'numberPixelX': numberPixelX, 'numberPixelY': numberPixelY}
# Pixel size
listPixelSizeXY = dictHeader['Pixel_size'].split(' ')
detector['pixelSizeX'] = float(listPixelSizeXY[0]) * 1000
detector['pixelSizeY'] = float(listPixelSizeXY[3]) * 1000
# Beam position
listBeamPosition = dictHeader['Beam_xy'].replace('(', ' ').replace(
')', ' ').replace(',', ' ').split()
detector['beamPositionX'] = float(listBeamPosition[0]) * \
detector['pixelSizeX']
detector['beamPositionY'] = float(listBeamPosition[1]) * \
detector['pixelSizeY']
distance = float(dictHeader['Detector_distance'].split(' ')[0]) * 1000
detector['distance'] = distance
detector['serialNumber'] = dictHeader['Detector:']
detector['name'] = detectorName
detector['type'] = detectorType
experimentalCondition['detector'] = detector
# Beam object
beam = {
'wavelength': float(dictHeader['Wavelength'].split(' ')[0]),
'exposureTime': float(dictHeader['Exposure_time'].split(' ')[0])
}
experimentalCondition['beam'] = beam
# Goniostat object
goniostat = {}
rotationAxisStart = float(dictHeader['Start_angle'].split(' ')[0])
oscillationWidth = float(dictHeader['Angle_increment'].split(' ')[0])
goniostat['rotationAxisStart'] = rotationAxisStart
goniostat['rotationAxisEnd'] = rotationAxisStart + oscillationWidth
goniostat['oscillationWidth'] = oscillationWidth
experimentalCondition['goniostat'] = goniostat
# Create the image object
image = {'path': imagePath}
if 'DateTime' in dictHeader:
image['date'] = dictHeader['DateTime']
imageNumber = UtilsImage.getImageNumber(imagePath)
image['number'] = imageNumber
subWedge = {
'experimentalCondition': experimentalCondition,
'image': [image]
}
return subWedge
def createThumbnail(
image,
format="jpg",
height=512,
width=512,
outputPath=None,
minLevel=0,
maxLevel=99.95,
dilatation=4,
workingDirectory=None,
outputFileName=None,
doResolutionRings=False,
pixelSizeX=None,
pixelSizeY=None,
beamPositionX=None,
beamPositionY=None,
distance=None,
wavelength=None,
):
imageFileName = os.path.basename(image)
imagePath = image
imageNameWithoutSuffix, imageSuffix = os.path.splitext(imageFileName)
if imageSuffix == ".h5":
imageNumber = UtilsImage.getImageNumber(image)
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
image, isFastMesh=True)
noTrials = 5
fabioImage = None
while noTrials > 0:
try:
fabioImage = fabio.openimage.openimage(h5MasterFilePath)
noTrials = 0
except Exception as e:
logger.debug("Error when trying to open {0}: {1}".format(
h5MasterFilePath, e))
logger.debug(
"Sleeping 5s and trying again, {0} trials left".format(
noTrials))
noTrials -= 1
time.sleep(5)
if fabioImage is None:
raise RuntimeError(
"Cannot open file {0} with fabio".format(h5MasterFilePath))
logger.debug("No frames: {0}".format(fabioImage.nframes))
if imageNumber < fabioImage.nframes:
numpyImage = fabioImage.getframe(imageNumber).data
else:
numpyImage = fabioImage.data
if numpyImage.dtype == numpy.dtype("uint32"):
numpyImage = numpy.where(numpyImage > 65536 * 65536 - 2, 0,
numpyImage)
else:
numpyImage = numpy.where(numpyImage > 256 * 256 - 2, 0,
numpyImage)
else:
fabioImage = fabio.openimage.openimage(image)
numpyImage = fabioImage.data
# Default format
suffix = "jpg"
pilFormat = "JPEG"
if format is not None:
if format.lower() == "png":
suffix = "png"
pilFormat = "PNG"
# The following code has been adapted from EDPluginExecThumbnail written by J.Kieffer
dtype = numpyImage.dtype
sortedArray = numpyImage.flatten()
sortedArray.sort()
numpyImage = numpy.maximum(numpyImage,
int(minLevel) * numpy.ones_like(numpyImage))
maxLevel = sortedArray[int(
round(float(maxLevel) * sortedArray.size / 100.0))]
if maxLevel < 25:
maxLevel = 25
numpyImage = numpy.minimum(numpyImage,
maxLevel * numpy.ones_like(numpyImage))
numpyImage = scipy.ndimage.morphology.grey_dilation(
numpyImage, (dilatation, dilatation))
mumpyImageFloat = (numpyImage.astype(numpy.float32)) / float(maxLevel)
numpyImageInt = (mumpyImageFloat * 255.0).astype(numpy.uint8)
# Check if we should do resolution rings
listResolution = []
if doResolutionRings:
delta = (height + width) / 2000
if delta < 1.0:
delta = 1.0
centreX = beamPositionX / pixelSizeX
centreY = beamPositionY / pixelSizeY
sizeY, sizeX = numpyImageInt.shape
averageSize = (sizeX + sizeY) / 2.0
yy, xx = numpy.mgrid[:sizeY, :sizeX]
circle = (xx - centreX)**2 + (yy - centreY)**2
for resolution in [1.0, 1.1, 1.2, 1.5, 2.0, 3.0, 4.0]:
import math
theta = math.asin(wavelength / (2 * resolution))
radius = math.tan(2 * theta) * distance / pixelSizeX
listResolution.append((resolution, radius / averageSize))
numpyImageInt = numpy.where(
numpy.logical_and(circle < (radius + delta)**2, circle >
(radius - delta)**2), 254, numpyImageInt)
pilOutputImage = ImageOps.invert(Image.fromarray(numpyImageInt, 'L'))
if height is not None and width is not None:
pilOutputImage = pilOutputImage.resize((width, height),
Image.ANTIALIAS)
width, height = pilOutputImage.size
for resolution, distance in listResolution:
centreX = width / 2
centreY = height / 2
textfont = ImageFont.truetype(
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
int(height / 30),
encoding="unic")
resolutionText = "{0} Å".format(resolution)
imageEditable = ImageDraw.Draw(pilOutputImage)
newDistance = distance * (height + width) / 2.0 / math.sqrt(2)
imageEditable.text((centreX + newDistance - width / 20,
centreY + newDistance - height / 20),
resolutionText,
0,
font=textfont)
if width * height > ImageFile.MAXBLOCK:
ImageFile.MAXBLOCK = width * height
if outputPath is None:
if outputFileName is None:
outputPath = os.path.join(
workingDirectory,
os.path.splitext(imageFileName)[0] + "." + suffix)
else:
outputPath = os.path.join(workingDirectory, outputFileName)
pilOutputImage.save(outputPath, pilFormat, quality=85, optimize=True)
logger.info("Output thumbnail path: %s" % outputPath)
return outputPath
def test_getImageNumber(self):
imageNumber = UtilsImage.getImageNumber(self.imageFileName)
self.assertEqual(imageNumber, 1)
def generateImageLinks(in_data, working_directory=None):
first_sub_wedge = in_data["subWedge"][0]
first_image_path = first_sub_wedge["image"][0]["path"]
prefix = UtilsImage.getPrefix(first_image_path)
suffix = UtilsImage.getSuffix(first_image_path)
if suffix == "h5":
lowest_xds_image_number = 1
highest_xds_image_number = 1
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
first_image_path, hasOverlap=True, isFastMesh=False)
h5MasterFile = os.path.basename((str(h5MasterFilePath)))
h5DataFile = os.path.basename((str(h5DataFilePath)))
list_image_link = [
[str(h5MasterFilePath), h5MasterFile],
[str(h5DataFilePath), h5DataFile],
]
if working_directory is not None:
os.symlink(str(h5MasterFilePath), h5MasterFile)
os.symlink(str(h5DataFilePath), h5DataFile)
template = h5MasterFile.replace("master", "??????")
lowest_xds_image_number = None
highest_xds_image_number = None
for subwedge in in_data["subWedge"]:
image_list = subwedge["image"]
for image_dict in image_list:
image_path = image_dict["path"]
image_number = UtilsImage.getImageNumber(image_path)
if lowest_xds_image_number is None or lowest_xds_image_number > image_number:
lowest_xds_image_number = image_number
if highest_xds_image_number is None or highest_xds_image_number < image_number:
highest_xds_image_number = image_number
else:
template = "%s_xdslink_?????.%s" % (prefix, suffix)
xds_lowest_image_number_global = 1
# First we have to find the smallest goniostat rotation axis start:
oscillation_start_min = 0
# Loop through the list of sub wedges
list_of_list = []
lowest_xds_image_number = None
highest_xds_image_number = None
list_spot_range = []
for sub_wedge in in_data["subWedge"]:
list_image_link = []
image_list = sub_wedge["image"]
goniostat = sub_wedge["experimentalCondition"]["goniostat"]
oscillation_start = goniostat["rotationAxisStart"]
oscillation_range = goniostat["oscillationWidth"]
# First find the lowest and highest image numbers
lowest_image_number = None
for image in image_list:
image_number = image["number"]
if lowest_image_number is None or image_number < lowest_image_number:
lowest_image_number = image_number
# Loop through the list of images
spot_range_min = None
spot_range_max = None
for image in image_list:
image_number = image["number"]
image_oscillation_start = (
oscillation_start +
(image_number - lowest_image_number) *
oscillation_range)
# if xdsLowestImageNumberGlobal is None:
# xdsLowestImageNumberGlobal = 1 + int((imageOscillationStart - oscillationStartMin) / oscillationRange)
xds_image_number = xds_lowest_image_number_global + int(
(image_oscillation_start - oscillation_start_min) /
oscillation_range + 0.5)
print(
xds_image_number,
image_oscillation_start,
oscillation_start_min,
oscillation_range,
)
source_path = image["path"]
target = "%s_xdslink_%05d.%s" % (prefix, xds_image_number,
suffix)
print([source_path, target])
list_image_link.append([source_path, target])
if working_directory is not None and not os.path.exists(
target):
os.symlink(source_path, target)
if (lowest_xds_image_number is None
or lowest_xds_image_number > xds_image_number):
lowest_xds_image_number = xds_image_number
if (highest_xds_image_number is None
or highest_xds_image_number < xds_image_number):
highest_xds_image_number = xds_image_number
if spot_range_min is None or spot_range_min > xds_image_number:
spot_range_min = xds_image_number
if spot_range_max is None or spot_range_max < xds_image_number:
spot_range_max = xds_image_number
list_spot_range.append([spot_range_min, spot_range_max])
list_of_list.append(list_image_link)
previous_exclude_data_range_max = 1
list_exclude_data_range = []
for spot_range_min, spot_range_max in list_spot_range:
if spot_range_min > previous_exclude_data_range_max + 1:
list_exclude_data_range.append(
[previous_exclude_data_range_max, spot_range_min - 1])
previous_exclude_data_range_max = spot_range_max + 1
dictImageLinks = {
"imageLink": list_of_list,
"spotRange": list_spot_range,
"dataRange": [lowest_xds_image_number, highest_xds_image_number],
"excludeDataRange": list_exclude_data_range,
"template": template,
}
return dictImageLinks
def createHdf5HeaderData(cls,
imagePath,
skipNumberOfImages=False,
hasOverlap=False,
isFastMesh=True):
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
pathlib.Path(imagePath),
isFastMesh=isFastMesh,
hasOverlap=hasOverlap)
# Waiting for file
timedOut, finalSize = UtilsPath.waitForFile(h5MasterFilePath,
expectedSize=2000000,
timeOut=DEFAULT_TIME_OUT)
if timedOut:
errorMessage = "Timeout when waiting for image %s" % imagePath
logger.error(errorMessage)
raise BaseException(errorMessage)
logger.info("Final size for {0}: {1}".format(h5MasterFilePath,
finalSize))
noTrialsLeft = 5
dictHeader = None
while noTrialsLeft > 0:
try:
dictHeader = cls.readHdf5Header(h5MasterFilePath)
noTrialsLeft = 0
except Exception as e:
logger.warning(
"Cannot read header from {0}, no trials left: {1}".format(
h5MasterFilePath, noTrialsLeft))
time.sleep(5)
noTrialsLeft -= 1
if dictHeader is None:
raise RuntimeError(
"Cannot read header from {0}!".format(h5MasterFilePath))
description = dictHeader["description"]
if "Eiger 4M" in description:
detectorName = "EIGER 4M"
detectorType = "eiger4m"
numberPixelX = 2070
numberPixelY = 2167
elif "eiger" in description.lower() and "16M" in description:
detectorName = "EIGER 16M"
detectorType = "eiger16m"
numberPixelX = 4148
numberPixelY = 4362
else:
raise RuntimeError(
"{0} cannot read image header from images with detector type {1}"
.format(cls.__class__.__name__, description))
# Image number
image_number = UtilsImage.getImageNumber(imagePath)
# Find out size of data set
prefix = str(h5MasterFilePath).split("master")[0]
listDataImage = []
noImages = 0
if not skipNumberOfImages:
for data in dictHeader["data"]:
dataFilePath = prefix + data + ".h5"
timedOut, finalSize = UtilsPath.waitForFile(
dataFilePath,
expectedSize=100000,
timeOut=DEFAULT_TIME_OUT)
if timedOut:
raise RuntimeError(
"Timeout waiting for file {0}".format(dataFilePath))
# listDataImage.append({
# 'path': dataFilePath
# })
f = h5py.File(dataFilePath, "r")
dataShape = f["entry"]["data"]["data"].shape
noImages += dataShape[0]
f.close()
experimentalCondition = {}
# Pixel size and beam position
detector = {
"numberPixelX":
int(numberPixelX),
"numberPixelY":
int(numberPixelY),
"pixelSizeX":
round(dictHeader["x_pixel_size"] * 1000, 3),
"pixelSizeY":
round(dictHeader["y_pixel_size"] * 1000, 3),
"beamPositionX":
round(
float(dictHeader["beam_center_x"] *
dictHeader["x_pixel_size"] * 1000),
3,
),
"beamPositionY":
round(
float(dictHeader["beam_center_y"] *
dictHeader["y_pixel_size"] * 1000),
3,
),
"distance":
round(float(dictHeader["detector_distance"]) * 1000, 3),
"serialNumber":
dictHeader["detector_number"],
"name":
detectorName,
"type":
detectorType,
}
experimentalCondition["detector"] = detector
# Beam object
beam = {
"wavelength": round(float(dictHeader["wavelength"]), 6),
"exposureTime": round(float(dictHeader["count_time"]), 6),
}
experimentalCondition["beam"] = beam
# Goniostat object
goniostat = {}
rotationAxisStart = round(float(dictHeader["omega_start"]), 4)
oscillationWidth = round(float(dictHeader["omega_range_average"]), 4)
# Offset for the image number
rotationAxisStart += (image_number - 1) * oscillationWidth
goniostat["rotationAxisStart"] = rotationAxisStart
goniostat["rotationAxisEnd"] = rotationAxisStart + oscillationWidth
goniostat["oscillationWidth"] = oscillationWidth
experimentalCondition["goniostat"] = goniostat
# Create the image object
image_dict = {
"path": imagePath,
"date": dictHeader["data_collection_date"],
"number": 1,
}
# imageNumber = UtilsImage.getImageNumber(imagePath)
# image['number'] = imageNumber
subWedge = {
"experimentalCondition": experimentalCondition,
"image": [image_dict],
}
return subWedge
def createCBFHeaderData(cls, imagePath):
# Waiting for file
timedOut, finalSize = UtilsPath.waitForFile(imagePath,
expectedSize=100000,
timeOut=DEFAULT_TIME_OUT)
if timedOut:
errorMessage = "Timeout when waiting for image %s" % imagePath
logger.error(errorMessage)
raise BaseException(errorMessage)
dictHeader = cls.readCBFHeader(imagePath)
detector = dictHeader["Detector:"]
if ("PILATUS 3M" in detector or "PILATUS3 2M" in detector
or "PILATUS 2M" in detector or "PILATUS2 3M" in detector):
detectorName = "PILATUS2 3M"
detectorType = "pilatus2m"
numberPixelX = 1475
numberPixelY = 1679
elif "PILATUS 6M" in detector or "PILATUS3 6M" in detector:
detectorName = "PILATUS2 6M"
detectorType = "pilatus6m"
numberPixelX = 2463
numberPixelY = 2527
elif "eiger" in detector.lower() and "4m" in detector.lower():
detectorName = "EIGER 4M"
detectorType = "eiger4m"
numberPixelX = 2070
numberPixelY = 2167
elif "eiger2" in detector.lower() and "16m" in detector.lower():
detectorName = "EIGER2 16M"
detectorType = "eiger16m"
numberPixelX = 4148
numberPixelY = 4362
else:
raise RuntimeError(
"{0} cannot read image header from images with dector type {1}"
.format(cls.__class__.__name__, detector))
experimentalCondition = {}
detector = {"numberPixelX": numberPixelX, "numberPixelY": numberPixelY}
# Pixel size
listPixelSizeXY = dictHeader["Pixel_size"].split(" ")
detector["pixelSizeX"] = float(listPixelSizeXY[0]) * 1000
detector["pixelSizeY"] = float(listPixelSizeXY[3]) * 1000
# Beam position
listBeamPosition = (dictHeader["Beam_xy"].replace("(", " ").replace(
")", " ").replace(",", " ").split())
detector["beamPositionX"] = float(
listBeamPosition[0]) * detector["pixelSizeX"]
detector["beamPositionY"] = float(
listBeamPosition[1]) * detector["pixelSizeY"]
distance = float(dictHeader["Detector_distance"].split(" ")[0]) * 1000
detector["distance"] = distance
detector["serialNumber"] = dictHeader["Detector:"]
detector["name"] = detectorName
detector["type"] = detectorType
experimentalCondition["detector"] = detector
# Beam object
beam = {
"wavelength": float(dictHeader["Wavelength"].split(" ")[0]),
"exposureTime": float(dictHeader["Exposure_time"].split(" ")[0]),
}
experimentalCondition["beam"] = beam
# Goniostat object
goniostat = {}
rotationAxisStart = float(dictHeader["Start_angle"].split(" ")[0])
oscillationWidth = float(dictHeader["Angle_increment"].split(" ")[0])
goniostat["rotationAxisStart"] = rotationAxisStart
goniostat["rotationAxisEnd"] = rotationAxisStart + oscillationWidth
goniostat["oscillationWidth"] = oscillationWidth
experimentalCondition["goniostat"] = goniostat
# Create the image object
image = {"path": imagePath}
if "DateTime" in dictHeader:
image["date"] = dictHeader["DateTime"]
imageNumber = UtilsImage.getImageNumber(imagePath)
image["number"] = imageNumber
subWedge = {
"experimentalCondition": experimentalCondition,
"image": [image]
}
return subWedge
def generateMOSFLMInData(cls, inData):
if "imagePath" in inData:
inDataReadImageHeader = {"imagePath": inData["imagePath"]}
readImageHeader = ReadImageHeader(inData=inDataReadImageHeader)
readImageHeader.execute()
listSubWedges = readImageHeader.outData["subWedge"]
elif "subWedge" in inData:
listSubWedges = inData["subWedge"]
else:
# Here we assume that we have inData in MOSFLM format...
return inData
firstSubWedge = listSubWedges[0]
experimentalCondition = firstSubWedge["experimentalCondition"]
detector = experimentalCondition["detector"]
beam = experimentalCondition["beam"]
goniostat = experimentalCondition["goniostat"]
detectorType = detector["type"]
if detectorType.startswith("pilatus"):
mosflmDetector = "PILATUS"
elif detectorType.startswith("eiger"):
mosflmDetector = "EIGER"
else:
raise RuntimeError("Unknown detecor type for MOSFLM: " +
detectorType)
listImage = firstSubWedge["image"]
firstImage = listImage[0]
firstPath = firstImage["path"]
mosflmInData = {
# The MOSFLM beam position convention is opposite to the XDS / CBF/ Dozor convention
"beam": {
"x": detector["beamPositionY"],
"y": detector["beamPositionX"]
},
"detector": {
"numberPixelX": detector["numberPixelX"],
"numberPixelY": detector["numberPixelY"],
"pixelSizeX": detector["pixelSizeX"],
"pixelSizeY": detector["pixelSizeY"],
"type": mosflmDetector
},
"directory": os.path.dirname(firstPath),
"distance": detector["distance"],
"template": UtilsImage.getTemplate(firstPath),
"wavelength": beam["wavelength"]
}
listMosflmImage = []
for subWedge in listSubWedges:
goniostat = subWedge["experimentalCondition"]["goniostat"]
rotationAxisStart = goniostat['rotationAxisStart']
oscillationWidth = goniostat['oscillationWidth']
imageIndex = 0
for image in subWedge["image"]:
number = UtilsImage.getImageNumber(image["path"])
rotationAxisStart = rotationAxisStart + oscillationWidth * imageIndex
rotationAxisEnd = rotationAxisStart + oscillationWidth
mosflmImage = {
"number": number,
"rotationAxisStart": rotationAxisStart,
"rotationAxisEnd": rotationAxisEnd
}
listMosflmImage.append(mosflmImage)
mosflmInData["image"] = listMosflmImage
return mosflmInData
def run(self, inData):
listImagePath = inData["imagePath"]
prefix = UtilsImage.getPrefix(listImagePath[0])
listSubWedge = self.getListSubWedge(inData)
diffractionPlan = inData.get("diffractionPlan", {})
sample = inData.get("sample", {})
# Check if flux is present
flux = None
absorbedDoseRate = None
experimentalCondition = inData.get("experimentalCondition", None)
if experimentalCondition is not None:
beam = experimentalCondition.get("beam", None)
if beam is not None:
flux = beam.get("flux", None)
# if not "size" in beam:
# beam["size"] = listSubWedge[0]["experimentalCondition"]["beam"]["size"]
beam["exposureTime"] = listSubWedge[0][
"experimentalCondition"]["beam"]["exposureTime"]
beam["wavelength"] = listSubWedge[0]["experimentalCondition"][
"beam"]["wavelength"]
# Convert images to CBF
firstImage = listSubWedge[0]["image"][0]["path"]
import os
import pprint
listH5ToCBF = []
suffix = os.path.splitext(firstImage)[1]
if suffix == ".h5":
for subWedge in listSubWedge:
imageList = subWedge["image"]
for image in imageList:
imagePath = image["path"]
hdf5ImageNumber = UtilsImage.getImageNumber(imagePath)
inDataH5ToCBF = {
"hdf5File": imagePath,
"hdf5ImageNumber": hdf5ImageNumber,
"imageNumber": 1,
"forcedOutputDirectory":
str(self.getWorkingDirectory()),
"forcedOutputImageNumber": hdf5ImageNumber,
}
h5ToCBF = H5ToCBFTask(inData=inDataH5ToCBF)
h5ToCBF.start()
listH5ToCBF.append((image, h5ToCBF))
# Join CBF creation
for image, h5ToCBF in listH5ToCBF:
h5ToCBF.join()
image["path"] = h5ToCBF.outData["outputCBFFile"]
pprint.pprint(listSubWedge)
# Start indexing
outDataIndexing, outDataGB, listSubWedge = self.indexing(
prefix, listSubWedge, self.getWorkingDirectory())
if outDataIndexing is not None and outDataGB is not None:
listXdsAsciiHkl, correctLp, bkgpixCbf = self.integration(
prefix, listSubWedge, outDataIndexing, outDataGB)
if listXdsAsciiHkl is not None:
# Check if Raddose should be run
estimateRadiationDamage = self.checkEstimateRadiationDamage(
inData, flux)
if estimateRadiationDamage:
# Check if forced space group
forcedSpaceGroup = None
numOperators = None
cell = None
if ("diffractionPlan" in inData and "forcedSpaceGroup"
in inData["diffractionPlan"]):
forcedSpaceGroup = inData["diffractionPlan"][
"forcedSpaceGroup"]
if forcedSpaceGroup is not None:
if forcedSpaceGroup != "":
forcedSpaceGroup = forcedSpaceGroup.replace(
" ", "")
numOperators = UtilsSymmetry.getNumberOfSymmetryOperatorsFromSpaceGroupName(
forcedSpaceGroup)
else:
forcedSpaceGroup = None
if forcedSpaceGroup is None:
# Get indexing space group IT number
if "resultIndexing" in outDataIndexing:
resultIndexing = outDataIndexing["resultIndexing"]
cell = resultIndexing["cell"]
if "spaceGroupNumber" in resultIndexing:
spaceGroupNumber = resultIndexing[
"spaceGroupNumber"]
numOperators = UtilsSymmetry.getNumberOfSymmetryOperatorsFromSpaceGroupITNumber(
spaceGroupNumber)
if numOperators is None:
raise RuntimeError(
"Error when trying to determine number of symmetry operators!"
)
chemicalComposition = self.getDefaultChemicalComposition(
cell, numOperators)
numberOfImages = self.getNumberOfImages(listSubWedge)
sample = inData["sample"]
inDataRaddose = {
"experimentalCondition": experimentalCondition,
"chemicalComposition": chemicalComposition,
"sample": sample,
"cell": cell,
"numberOfImages": numberOfImages,
"numOperators": numOperators,
}
# import pprint
# pprint.pprint(inDataRaddose)
raddose = Raddose(inData=inDataRaddose,
workingDirectorySuffix=prefix)
raddose.execute()
if raddose.isSuccess():
absorbedDoseRate = raddose.outData["absorbedDoseRate"]
inDataBest = {
"diffractionPlan": diffractionPlan,
"sample": sample,
"subWedge": listSubWedge,
"xdsAsciiHkl": listXdsAsciiHkl,
"bkgpixCbf": bkgpixCbf,
"correctLp": correctLp,
"crystalAbsorbedDoseRate": absorbedDoseRate,
}
bestTask = Best(inData=inDataBest,
workingDirectorySuffix=prefix)
bestTask.execute()
def run(self, inData):
beamline = inData.get('beamline', None)
doSubmit = inData.get('doSubmit', False)
doDozorm = inData.get('doDozorm', False)
batchSize = inData.get('batchSize', 1)
doDistlSignalStrength = inData.get('doDistlSignalStrength', False)
doIndexing = inData.get('doIndexing', False)
if crystFelImportFailed:
doCrystfel = False
else:
doCrystfel = inData.get('doCrystfel', True)
isFastMesh = inData.get('fastMesh', False)
# Loop through all the incoming reference images
listImage = inData.get('image', [])
if len(listImage) == 0:
directory = pathlib.Path(inData['directory'])
template = inData['template']
startNo = inData['startNo']
endNo = inData['endNo']
listImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = directory / imageName
listImage.append(str(imagePath))
else:
firstImage = listImage[0]
lastImage = listImage[-1]
directory = pathlib.Path(firstImage).parent
template = UtilsImage.getTemplate(firstImage)
startNo = UtilsImage.getImageNumber(firstImage)
endNo = UtilsImage.getImageNumber(lastImage)
outData = dict()
listImageQualityIndicators = []
listcrystfel_output = []
inDataWaitFile = {}
listDistlTask = []
listDozorTask = []
listCrystFELTask = []
listOfImagesInBatch = []
listOfAllBatches = []
listOfAllH5Files = []
listControlDozorAllFile = []
indexBatch = 0
self.listH5FilePath = []
detectorType = None
# Configurations
minImageSize = UtilsConfig.get(
self, 'minImageSize', defaultValue=DEFAULT_MIN_IMAGE_SIZE)
waitFileTimeOut = UtilsConfig.get(
self, 'waitFileTimeOut', defaultValue=DEFAULT_WAIT_FILE_TIMEOUT)
# Process data in batches
for image in listImage:
listOfImagesInBatch.append(pathlib.Path(image))
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if UtilsImage.getSuffix(pathlib.Path(listImage[0])) == 'h5':
for image in listImage:
listOfAllH5Files.append(pathlib.Path(image))
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and Crystfel (if required) in parallel
#
# Check if we should run CrystFEL:
for listOfImagesInBatch in listOfAllBatches:
listOfH5FilesInBatch = []
for imagePath in listOfImagesInBatch:
# First wait for images
self.waitForImagePath(
imagePath=imagePath,
batchSize=batchSize,
isFastMesh=isFastMesh,
minImageSize=minImageSize,
waitFileTimeOut=waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch
)
if not self.isFailure():
# Determine start and end image no
pathToFirstImage = listOfImagesInBatch[0]
pathToLastImage = listOfImagesInBatch[-1]
batchStartNo = UtilsImage.getImageNumber(pathToFirstImage)
batchEndNo = UtilsImage.getImageNumber(pathToLastImage)
# Run Control Dozor
inDataControlDozor = {
'template': template,
'directory': directory,
'startNo': batchStartNo,
'endNo': batchEndNo,
'batchSize': batchSize,
'doSubmit': doSubmit,
'doDozorm': doDozorm
}
if beamline is not None:
inDataControlDozor["beamline"] = beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix='{0:04d}_{1:04d}'.format(batchStartNo, batchEndNo))
controlDozor.start()
listDozorTask.append((controlDozor, inDataControlDozor,
list(listOfImagesInBatch), listOfH5FilesInBatch))
# Check if we should run distl.signalStrength
if doDistlSignalStrength:
for image in listOfImagesInBatch:
inDataDistl = {
'referenceImage': str(image)
}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=UtilsImage.getImageNumber(image))
distlTask.start()
listDistlTask.append((image, distlTask))
if not self.isFailure():
# listIndexing = []
listDistlResult = []
# Synchronize all image quality indicator plugins and upload to ISPyB
for (image, distlTask) in listDistlTask:
imageQualityIndicators = {}
if distlTask is not None:
distlTask.join()
if distlTask.isSuccess():
outDataDistl = distlTask.outData
if outDataDistl is not None:
imageQualityIndicators = outDataDistl['imageQualityIndicators']
imageQualityIndicators['image'] = str(image)
listDistlResult.append(imageQualityIndicators)
# self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (controlDozor, inDataControlDozor, listBatch, listH5FilePath) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData['imageQualityIndicators']) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0].name
lastImage = listBatch[-1].name
logger.warning("No dozor results! Re-executing Dozor for" +
" images {0} to {1}".format(firstImage, lastImage))
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(controlDozor.outData['imageQualityIndicators'])
if detectorType is None:
detectorType = controlDozor.outData['detectorType']
if doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor['image'] == distlResult['image']:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if doDozorm:
listControlDozorAllFile.append(controlDozor.outData['dozorAllFile'])
if not self.isFailure() and doCrystfel:
# Select only the strongest images to be run by CrystFEL
listIndicatorsSorted = sorted(listImageQualityIndicators, key=lambda k: k['dozorScore'])[::-1]
listForCrystFEL = [k['image'] for k in listIndicatorsSorted[0:min(200, len(listIndicatorsSorted))]]
batchSize = 20
if len(listForCrystFEL) % batchSize == 0:
file_chunk = int(len(listForCrystFEL) / batchSize)
else:
file_chunk = int(len(listForCrystFEL) / batchSize) + 1
for jj in range(file_chunk):
start = batchSize * jj
stop = batchSize * (jj + 1)
try:
images = listForCrystFEL[start:stop]
except IndexError:
stop = start + (len(listForCrystFEL) - stop)
images = listForCrystFEL[start:stop]
inDataCrystFEL = {
'doCBFtoH5': False,
'cbfFileInfo': {
'directory': directory,
'template': template,
'batchSize': batchSize,
'listofImages': images
},
'doSubmit': doSubmit
}
crystfel = ExeCrystFEL(inData=inDataCrystFEL)
crystfel.start()
listCrystFELTask.append(crystfel)
masterstream = str(self.getWorkingDirectory() / 'alltogether.stream')
if not self.isFailure():
for crystfel in listCrystFELTask:
crystfel.join()
if crystfel.isSuccess():
catcommand = "cat %s >> %s" % (crystfel.outData['streamfile'], masterstream)
AutoCrystFEL.run_as_command(catcommand)
if not self.isFailure() and os.path.exists(masterstream):
crystfel_outdata = AutoCrystFEL.report_stats(masterstream)
crystfel_outdata['number of DozorHits'] = len(listForCrystFEL)
AutoCrystFEL.write_cell_file(crystfel_outdata)
listcrystfel_output.append(crystfel_outdata)
else:
logger.error("CrystFEL did not run properly")
# Assemble all controlDozorAllFiles into one
if doDozorm:
imageQualityIndicatorsDozorAllFile = str(self.getWorkingDirectory() / "dozor_all")
os.system('touch {0}'.format(imageQualityIndicatorsDozorAllFile))
for controlDozorAllFile in listControlDozorAllFile:
command = 'cat ' + controlDozorAllFile + ' >> ' + imageQualityIndicatorsDozorAllFile
os.system(command)
outData["dozorAllFile"] = imageQualityIndicatorsDozorAllFile
outData['imageQualityIndicators'] = listImageQualityIndicators
outData['crystfel_results'] = listcrystfel_output
return outData