def init(self, inData):
self.beamline = inData.get("beamline", None)
self.doSubmit = inData.get("doSubmit", False)
self.doDozorM = inData.get("doDozorM", False)
self.doDistlSignalStrength = inData.get("doDistlSignalStrength", False)
self.isFastMesh = inData.get("fastMesh", False)
self.listImage = inData.get("image", [])
self.batchSize = inData.get("batchSize", 1)
self.doIspybUpload = inData.get("doIspybUpload", False)
self.dataCollectionId = inData.get("dataCollectionId", None)
# Configurations
self.minImageSize = UtilsConfig.get(
self, "minImageSize", defaultValue=DEFAULT_MIN_IMAGE_SIZE
)
self.waitFileTimeOut = UtilsConfig.get(
self, "waitFileTimeOut", defaultValue=DEFAULT_WAIT_FILE_TIMEOUT
)
def addFileHandler(logger):
logPath = UtilsConfig.get("Logging", "log_file_path")
if logPath is not None:
if "DATE" in logPath:
logPath = logPath.replace(
"DATE", time.strftime("%Y-%m-%d", time.localtime(time.time())))
if not os.path.exists(os.path.dirname(logPath)):
os.makedirs(os.path.dirname(logPath))
maxBytes = int(UtilsConfig.get("Logging", "log_file_maxbytes", 1e7))
backupCount = int(UtilsConfig.get("Logging", "log_file_backupCount",
0))
fileHandler = logging.handlers.RotatingFileHandler(
logPath, maxBytes=maxBytes, backupCount=backupCount)
logFileFormat = UtilsConfig.get("Logging", "log_file_format")
if logFileFormat is None:
logFileFormat = (
"%(asctime)s %(module)-20s %(funcName)-15s %(levelname)-8s %(message)s"
)
formatter = logging.Formatter(logFileFormat)
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
def run(self, inData):
# Wait for file if it's not already on disk'
if "file" not in inData:
raise BaseException("No expected file path in input!")
filePath = pathlib.Path(self.inData["file"])
expectedSize = inData.get("expectedSize", None)
configTimeOut = UtilsConfig.get(self, "timeOut", DEFAULT_TIMEOUT)
timeOut = inData.get("timeOut", configTimeOut)
hasTimedOut, finalSize = UtilsPath.waitForFile(
filePath, expectedSize=expectedSize, timeOut=timeOut)
outData = {"timedOut": hasTimedOut}
if finalSize is not None:
outData["finalSize"] = finalSize
return outData
def run(self, inData):
outData = {}
input_file = inData['input_file']
output_file = inData['output_file']
symoplib = UtilsConfig.get('CCP4', 'symoplib')
ccp4setup = UtilsConfig.get('CCP4', 'ccp4setup')
if ccp4setup is None:
commandLine = ""
else:
commandLine = ". " + ccp4setup + '\n'
if symoplib is None:
commandLine += 'aimless HKLIN {0} HKLOUT {1}'.format(
input_file, output_file)
else:
commandLine += 'aimless HKLIN {0} HKLOUT {1} SYMINFO {2}'.format(
input_file, output_file, symoplib)
logger.info("Command line: {0}".format(commandLine))
start_image = inData['start_image']
end_image = inData['end_image']
projectName = inData.get('dataCollectionID', 'EDNA_proc')
resolution = inData.get('res', 0.0)
anom = inData['anom']
listCommand = [
'bins 15',
'run 1 batch {0} to {1}'.format(start_image, end_image),
'name run 1 project {0} crystal DEFAULT dataset NATIVE'.format(projectName),
'scales constant',
'resolution 50 {0}'.format(resolution),
'cycles 100',
'anomalous {0}'.format('ON' if anom else 'OFF'),
'output MERGED UNMERGED',
'END'
]
self.setLogFileName('aimless.log')
self.runCommandLine(commandLine, listCommand=listCommand)
outData['isSuccess'] = os.path.exists(output_file)
return outData
def setLoggingLevel(logger, level):
if level is None:
level = UtilsConfig.get("Logging", "level")
if level is None:
level = "INFO"
if level == "DEBUG":
loggingLevel = logging.DEBUG
elif level == "INFO":
loggingLevel = logging.INFO
elif level == "WARNING":
loggingLevel = logging.WARNING
elif level == "ERROR":
loggingLevel = logging.ERROR
elif level == "CRITICAL":
loggingLevel = logging.CRITICAL
elif level == "FATAL":
loggingLevel = logging.FATAL
else:
raise RuntimeError('Unknown logging level: "{0}"'.format(level))
logger.setLevel(loggingLevel)
def getLogger(level=None):
if level is None:
level = UtilsConfig.get('Logging', 'level')
if level is None:
level = 'INFO'
# Check if graylog handler already added:
logger = logging.getLogger('edna2')
hasGraylogHandler = False
hasStreamHandler = False
hasFileHandler = False
for handler in logger.handlers:
if isinstance(handler, graypy.GELFUDPHandler):
hasGraylogHandler = True
elif isinstance(handler, logging.handlers.RotatingFileHandler):
hasFileHandler = True
elif isinstance(handler, logging.StreamHandler):
hasStreamHandler = True
if not hasGraylogHandler:
server = UtilsConfig.get('Logging', 'graylog_server')
port = UtilsConfig.get('Logging', 'graylog_port')
if server is not None and port is not None:
graylogHandler = graypy.GELFUDPHandler(server, int(port))
logger.addHandler(graylogHandler)
if not hasStreamHandler:
streamHandler = logging.StreamHandler()
logFileFormat = '%(asctime)s %(levelname)-8s %(message)s'
formatter = logging.Formatter(logFileFormat)
streamHandler.setFormatter(formatter)
logger.addHandler(streamHandler)
if not hasFileHandler:
logPath = UtilsConfig.get('Logging', 'log_file_path')
if logPath is not None:
if not os.path.exists(os.path.dirname(logPath)):
os.makedirs(os.path.dirname(logPath))
maxBytes = int(UtilsConfig.get('Logging', 'log_file_maxbytes', 1e6))
backupCount = int(UtilsConfig.get('Logging', 'log_file_backupCount', 10))
fileHandler = logging.handlers.RotatingFileHandler(
logPath, maxBytes=maxBytes, backupCount=backupCount)
logFileFormat = UtilsConfig.get('Logging', 'log_file_format')
if logFileFormat is None:
logFileFormat = '%(asctime)s %(levelname)-8s %(message)s'
formatter = logging.Formatter(logFileFormat)
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
# Set level
if level == 'DEBUG':
loggingLevel = logging.DEBUG
elif level == 'INFO':
loggingLevel = logging.INFO
elif level == 'WARNING':
loggingLevel = logging.WARNING
elif level == 'ERROR':
loggingLevel = logging.ERROR
elif level == 'CRITICAL':
loggingLevel = logging.CRITICAL
elif level == 'FATAL':
loggingLevel = logging.FATAL
else:
raise RuntimeError('Unknown logging level: "{0}"'.format(level))
logger.setLevel(loggingLevel)
return logger
def addGrayLogHandler(logger):
server = UtilsConfig.get("Logging", "graylog_server")
port = UtilsConfig.get("Logging", "graylog_port")
if server is not None and port is not None:
graylogHandler = graypy.GELFUDPHandler(server, int(port))
logger.addHandler(graylogHandler)
def generateXDS_INP(inData):
"""
This method creates a list of XDS.INP commands
"""
# Take the first sub webge in input as reference
firstSubwedge = inData["subWedge"][0]
listImage = firstSubwedge["image"]
image = listImage[0]
experimentalCondition = firstSubwedge["experimentalCondition"]
detector = experimentalCondition["detector"]
dictXDSDetector = XDSTask.getXDSDetector(detector)
beam = experimentalCondition["beam"]
goniostat = experimentalCondition["goniostat"]
distance = round(detector["distance"], 3)
wavelength = round(beam["wavelength"], 3)
oscRange = goniostat["oscillationWidth"]
startAngle = goniostat["rotationAxisStart"] - int(
goniostat["rotationAxisStart"])
listXDS_INP = [
"OVERLOAD=10048500",
"DIRECTION_OF_DETECTOR_X-AXIS={0}".format(
UtilsConfig.get("XDSTask", "DIRECTION_OF_DETECTOR_X-AXIS")),
"DIRECTION_OF_DETECTOR_Y-AXIS={0}".format(
UtilsConfig.get("XDSTask", "DIRECTION_OF_DETECTOR_Y-AXIS")),
"ROTATION_AXIS={0}".format(
UtilsConfig.get("XDSTask", "ROTATION_AXIS")),
"INCIDENT_BEAM_DIRECTION={0}".format(
UtilsConfig.get("XDSTask", "INCIDENT_BEAM_DIRECTION")),
"NX={0} NY={1} QX={2} QY={2}".format(dictXDSDetector["nx"],
dictXDSDetector["ny"],
dictXDSDetector["pixel"]),
"ORGX={0} ORGY={1}".format(dictXDSDetector["orgX"],
dictXDSDetector["orgY"]),
"DETECTOR={0} MINIMUM_VALID_PIXEL_VALUE={1} OVERLOAD={2}".format(
dictXDSDetector["name"],
dictXDSDetector["minimumValidPixelValue"],
dictXDSDetector["overload"],
),
"SENSOR_THICKNESS={0}".format(dictXDSDetector["sensorThickness"]),
"TRUSTED_REGION={0} {1}".format(
dictXDSDetector["trustedRegion"][0],
dictXDSDetector["trustedRegion"][1]),
]
# for trustedRegion in dictXDSDetector["untrustedRectangle"]:
# listXDS_INP.append('UNTRUSTED_RECTANGLE={0} {1} {2} {3}'.format(
# trustedRegion[0], trustedRegion[1],
# trustedRegion[2],trustedRegion[3]
# ))
listXDS_INP += [
"DETECTOR_DISTANCE={0}".format(distance),
"X-RAY_WAVELENGTH={0}".format(wavelength),
"OSCILLATION_RANGE={0}".format(oscRange),
"STARTING_ANGLE={0}".format(startAngle),
"INDEX_QUALITY= 0.25",
]
if "spaceGroupNumber" in inData:
spaceGroupNumber = inData["spaceGroupNumber"]
cell = inData["cell"]
unitCellConstants = "{a} {b} {c} {alpha} {beta} {gamma}".format(
**cell)
listXDS_INP += [
"SPACE_GROUP_NUMBER={0}".format(spaceGroupNumber),
"UNIT_CELL_CONSTANTS={0}".format(unitCellConstants),
]
if (image["path"].endswith("h5")
and UtilsConfig.get("XDSTask", "LIB") is not None):
listXDS_INP += [
"LIB= {0}".format(UtilsConfig.get("XDSTask", "LIB"))
]
return listXDS_INP
def generateMOSFLMCommands(self, inData, workingDirectory):
"""
This method creates a list of MOSFLM commands
"""
detector = inData['detector']
detectorType = detector['type']
detectorCommand = detectorType
# Check if reversephi and omega are configured
omega = UtilsConfig.get('MOSFLM', 'omega')
if omega is not None:
detectorCommand += ' OMEGA {0}'.format(omega)
reversePhi = UtilsConfig.get('MOSFLM', 'reversePhi')
if reversePhi is not None:
detectorCommand += ' REVERSEPHI'
listCommand = [
'WAVELENGTH {0}'.format(inData['wavelength']),
'DISTANCE {0}'.format(inData['distance']),
'BEAM {0} {1}'.format(inData['beam']['x'], inData['beam']['y']),
'DIRECTORY {0}'.format(inData['directory']),
'TEMPLATE {0}'.format(inData['template'])
]
if 'symmetry' in inData:
listCommand.append('SYMMETRY {0}'.format(inData['symmetry']))
if 'mosaicity' in inData:
listCommand.append('MOSAIC {0}'.format(inData['mosaicity']))
newmatFileName = self.getNewmatFileName()
newmatMatrix = inData.get('matrix', None)
if newmatMatrix is not None:
matrixFileName = self.getMatrixFileName()
newmatPath = workingDirectory / matrixFileName
self.writeNewmat(newmatMatrix, newmatPath)
listCommand.append('MATRIX ' + matrixFileName)
# Add exclude regions if Pilatus
if detectorType == 'PILATUS':
listCommand.append('DETECTOR {0}'.format(detectorCommand))
if detector['numberPixelX'] == 1475 and \
detector['numberPixelY'] == 1679:
# Pilatus 2M
listCommand.append(
'LIMITS EXCLUDE 0.0 83.76 288.96 85.14')
listCommand.append(
'LIMITS EXCLUDE 0.0 168.73 288.96 170.10')
listCommand.append(
'LIMITS EXCLUDE 33.54 0.0 36.63 253.87')
listCommand.append(
'LIMITS EXCLUDE 70.00 0.0 73.1 253.87')
listCommand.append(
'LIMITS EXCLUDE 106.46 0.0 109.56 253.87')
listCommand.append(
'LIMITS EXCLUDE 142.93 0.0 146.02 253.87')
listCommand.append(
'LIMITS EXCLUDE 179.39 0.0 182.49 253.87')
listCommand.append(
'LIMITS EXCLUDE 215.86 0.0 218.95 253.87')
listCommand.append(
'LIMITS EXCLUDE 252.32 0.0 255.42 253.87')
elif detector['numberPixelX'] == 2463 and \
detector['numberPixelY'] == 2527:
# Pilatus 6M
listCommand.append('LIMITS EXCLUDE 0.0 338.77 434.6 340.24')
listCommand.append('LIMITS EXCLUDE 0.0 253.80 434.6 255.28')
listCommand.append('LIMITS EXCLUDE 0.0 168.83 434.6 170.21')
listCommand.append('LIMITS EXCLUDE 0.0 83.86 434.6 85.24')
listCommand.append('LIMITS EXCLUDE 398.18 0.0 401.28 423.6')
listCommand.append('LIMITS EXCLUDE 361.72 0.0 364.81 423.6')
listCommand.append('LIMITS EXCLUDE 325.25 0.0 328.35 423.6')
listCommand.append('LIMITS EXCLUDE 288.79 0.0 291.88 423.6')
listCommand.append('LIMITS EXCLUDE 252.32 0.0 255.42 423.6')
listCommand.append('LIMITS EXCLUDE 215.86 0.0 218.96 423.6')
listCommand.append('LIMITS EXCLUDE 179.40 0.0 182.49 423.6')
listCommand.append('LIMITS EXCLUDE 142.93 0.0 145.86 423.6')
listCommand.append('LIMITS EXCLUDE 106.47 0.0 109.56 423.6')
listCommand.append('LIMITS EXCLUDE 70.00 0.0 73.10 423.6')
listCommand.append('LIMITS EXCLUDE 33.54 0.0 36.64 423.6')
# Check if raster is configured
raster = UtilsConfig.get('MOSFLM', 'raster')
if raster is not None:
listCommand.append('RASTER {0}'.format(raster))
# Check if polarization is configured
polarization = UtilsConfig.get('MOSFLM', 'polarization')
if polarization is not None:
listCommand.append('POLARIZATION {0}'.format(polarization))
return listCommand
def generateXDS_INP(inData):
"""
This method creates a list of XDS.INP commands
"""
# Take the first sub webge in input as reference
firstSubwedge = inData["subWedge"][0]
listImage = firstSubwedge['image']
image = listImage[0]
experimentalCondition = firstSubwedge['experimentalCondition']
detector = experimentalCondition['detector']
dictXDSDetector = XDSTask.getXDSDetector(detector)
beam = experimentalCondition['beam']
goniostat = experimentalCondition['goniostat']
distance = round(detector['distance'], 3)
wavelength = round(beam['wavelength'], 3)
oscRange = goniostat['oscillationWidth']
startAngle = goniostat['rotationAxisStart'] - int(
goniostat['rotationAxisStart'])
dataRange = '1 360'
listXDS_INP = [
'OVERLOAD=10048500', 'DIRECTION_OF_DETECTOR_X-AXIS={0}'.format(
UtilsConfig.get('XDSTask', 'DIRECTION_OF_DETECTOR_X-AXIS')),
'DIRECTION_OF_DETECTOR_Y-AXIS={0}'.format(
UtilsConfig.get('XDSTask', 'DIRECTION_OF_DETECTOR_Y-AXIS')),
'ROTATION_AXIS={0}'.format(
UtilsConfig.get('XDSTask', 'ROTATION_AXIS')),
'INCIDENT_BEAM_DIRECTION={0}'.format(
UtilsConfig.get('XDSTask', 'INCIDENT_BEAM_DIRECTION')),
'NX={0} NY={1} QX={2} QY={2}'.format(dictXDSDetector["nx"],
dictXDSDetector["ny"],
dictXDSDetector["pixel"]),
'ORGX={0} ORGY={1}'.format(dictXDSDetector["orgX"],
dictXDSDetector["orgY"]),
'DETECTOR={0} MINIMUM_VALID_PIXEL_VALUE={1} OVERLOAD={2}'.format(
dictXDSDetector["name"],
dictXDSDetector["minimumValidPixelValue"],
dictXDSDetector["overload"]),
'SENSOR_THICKNESS={0}'.format(dictXDSDetector["sensorThickness"]),
'TRUSTED_REGION={0} {1}'.format(
dictXDSDetector["trustedRegion"][0],
dictXDSDetector["trustedRegion"][1])
]
# for trustedRegion in dictXDSDetector["untrustedRectangle"]:
# listXDS_INP.append('UNTRUSTED_RECTANGLE={0} {1} {2} {3}'.format(
# trustedRegion[0], trustedRegion[1],
# trustedRegion[2],trustedRegion[3]
# ))
listXDS_INP += [
'DETECTOR_DISTANCE={0}'.format(distance),
'X-RAY_WAVELENGTH={0}'.format(wavelength),
'OSCILLATION_RANGE={0}'.format(oscRange),
'STARTING_ANGLE={0}'.format(startAngle), 'INDEX_QUALITY= 0.25'
]
if "spaceGroupNumber" in inData:
spaceGroupNumber = inData["spaceGroupNumber"]
cell = inData["cell"]
unitCellConstants = "{a} {b} {c} {alpha} {beta} {gamma}".format(
**cell)
listXDS_INP += [
'SPACE_GROUP_NUMBER={0}'.format(spaceGroupNumber),
'UNIT_CELL_CONSTANTS={0}'.format(unitCellConstants)
]
return listXDS_INP
def run(self, inData):
beamline = inData.get('beamline', None)
doSubmit = inData.get('doSubmit', False)
doDozorm = inData.get('doDozorm', False)
batchSize = inData.get('batchSize', 1)
doDistlSignalStrength = inData.get('doDistlSignalStrength', False)
doIndexing = inData.get('doIndexing', False)
if crystFelImportFailed:
doCrystfel = False
else:
doCrystfel = inData.get('doCrystfel', True)
isFastMesh = inData.get('fastMesh', False)
# Loop through all the incoming reference images
listImage = inData.get('image', [])
if len(listImage) == 0:
directory = pathlib.Path(inData['directory'])
template = inData['template']
startNo = inData['startNo']
endNo = inData['endNo']
listImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = directory / imageName
listImage.append(str(imagePath))
else:
firstImage = listImage[0]
lastImage = listImage[-1]
directory = pathlib.Path(firstImage).parent
template = UtilsImage.getTemplate(firstImage)
startNo = UtilsImage.getImageNumber(firstImage)
endNo = UtilsImage.getImageNumber(lastImage)
outData = dict()
listImageQualityIndicators = []
listcrystfel_output = []
inDataWaitFile = {}
listDistlTask = []
listDozorTask = []
listCrystFELTask = []
listOfImagesInBatch = []
listOfAllBatches = []
listOfAllH5Files = []
listControlDozorAllFile = []
indexBatch = 0
self.listH5FilePath = []
detectorType = None
# Configurations
minImageSize = UtilsConfig.get(
self, 'minImageSize', defaultValue=DEFAULT_MIN_IMAGE_SIZE)
waitFileTimeOut = UtilsConfig.get(
self, 'waitFileTimeOut', defaultValue=DEFAULT_WAIT_FILE_TIMEOUT)
# Process data in batches
for image in listImage:
listOfImagesInBatch.append(pathlib.Path(image))
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if UtilsImage.getSuffix(pathlib.Path(listImage[0])) == 'h5':
for image in listImage:
listOfAllH5Files.append(pathlib.Path(image))
#
# Loop over batches:
# - Wait for all files in batch
# - Run Dozor and Crystfel (if required) in parallel
#
# Check if we should run CrystFEL:
for listOfImagesInBatch in listOfAllBatches:
listOfH5FilesInBatch = []
for imagePath in listOfImagesInBatch:
# First wait for images
self.waitForImagePath(
imagePath=imagePath,
batchSize=batchSize,
isFastMesh=isFastMesh,
minImageSize=minImageSize,
waitFileTimeOut=waitFileTimeOut,
listofH5FilesInBatch=listOfH5FilesInBatch
)
if not self.isFailure():
# Determine start and end image no
pathToFirstImage = listOfImagesInBatch[0]
pathToLastImage = listOfImagesInBatch[-1]
batchStartNo = UtilsImage.getImageNumber(pathToFirstImage)
batchEndNo = UtilsImage.getImageNumber(pathToLastImage)
# Run Control Dozor
inDataControlDozor = {
'template': template,
'directory': directory,
'startNo': batchStartNo,
'endNo': batchEndNo,
'batchSize': batchSize,
'doSubmit': doSubmit,
'doDozorm': doDozorm
}
if beamline is not None:
inDataControlDozor["beamline"] = beamline
controlDozor = ControlDozor(
inDataControlDozor,
workingDirectorySuffix='{0:04d}_{1:04d}'.format(batchStartNo, batchEndNo))
controlDozor.start()
listDozorTask.append((controlDozor, inDataControlDozor,
list(listOfImagesInBatch), listOfH5FilesInBatch))
# Check if we should run distl.signalStrength
if doDistlSignalStrength:
for image in listOfImagesInBatch:
inDataDistl = {
'referenceImage': str(image)
}
distlTask = DistlSignalStrengthTask(
inData=inDataDistl,
workingDirectorySuffix=UtilsImage.getImageNumber(image))
distlTask.start()
listDistlTask.append((image, distlTask))
if not self.isFailure():
# listIndexing = []
listDistlResult = []
# Synchronize all image quality indicator plugins and upload to ISPyB
for (image, distlTask) in listDistlTask:
imageQualityIndicators = {}
if distlTask is not None:
distlTask.join()
if distlTask.isSuccess():
outDataDistl = distlTask.outData
if outDataDistl is not None:
imageQualityIndicators = outDataDistl['imageQualityIndicators']
imageQualityIndicators['image'] = str(image)
listDistlResult.append(imageQualityIndicators)
# self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
for (controlDozor, inDataControlDozor, listBatch, listH5FilePath) in listDozorTask:
controlDozor.join()
# Check that we got at least one result
if len(controlDozor.outData['imageQualityIndicators']) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = listBatch[0].name
lastImage = listBatch[-1].name
logger.warning("No dozor results! Re-executing Dozor for" +
" images {0} to {1}".format(firstImage, lastImage))
time.sleep(5)
controlDozor = ControlDozor(inDataControlDozor)
controlDozor.execute()
listOutDataControlDozor = list(controlDozor.outData['imageQualityIndicators'])
if detectorType is None:
detectorType = controlDozor.outData['detectorType']
if doDistlSignalStrength:
for outDataControlDozor in listOutDataControlDozor:
for distlResult in listDistlResult:
if outDataControlDozor['image'] == distlResult['image']:
imageQualityIndicators = dict(outDataControlDozor)
imageQualityIndicators.update(distlResult)
listImageQualityIndicators.append(imageQualityIndicators)
else:
listImageQualityIndicators += listOutDataControlDozor
# Check if dozorm
if doDozorm:
listControlDozorAllFile.append(controlDozor.outData['dozorAllFile'])
if not self.isFailure() and doCrystfel:
# Select only the strongest images to be run by CrystFEL
listIndicatorsSorted = sorted(listImageQualityIndicators, key=lambda k: k['dozorScore'])[::-1]
listForCrystFEL = [k['image'] for k in listIndicatorsSorted[0:min(200, len(listIndicatorsSorted))]]
batchSize = 20
if len(listForCrystFEL) % batchSize == 0:
file_chunk = int(len(listForCrystFEL) / batchSize)
else:
file_chunk = int(len(listForCrystFEL) / batchSize) + 1
for jj in range(file_chunk):
start = batchSize * jj
stop = batchSize * (jj + 1)
try:
images = listForCrystFEL[start:stop]
except IndexError:
stop = start + (len(listForCrystFEL) - stop)
images = listForCrystFEL[start:stop]
inDataCrystFEL = {
'doCBFtoH5': False,
'cbfFileInfo': {
'directory': directory,
'template': template,
'batchSize': batchSize,
'listofImages': images
},
'doSubmit': doSubmit
}
crystfel = ExeCrystFEL(inData=inDataCrystFEL)
crystfel.start()
listCrystFELTask.append(crystfel)
masterstream = str(self.getWorkingDirectory() / 'alltogether.stream')
if not self.isFailure():
for crystfel in listCrystFELTask:
crystfel.join()
if crystfel.isSuccess():
catcommand = "cat %s >> %s" % (crystfel.outData['streamfile'], masterstream)
AutoCrystFEL.run_as_command(catcommand)
if not self.isFailure() and os.path.exists(masterstream):
crystfel_outdata = AutoCrystFEL.report_stats(masterstream)
crystfel_outdata['number of DozorHits'] = len(listForCrystFEL)
AutoCrystFEL.write_cell_file(crystfel_outdata)
listcrystfel_output.append(crystfel_outdata)
else:
logger.error("CrystFEL did not run properly")
# Assemble all controlDozorAllFiles into one
if doDozorm:
imageQualityIndicatorsDozorAllFile = str(self.getWorkingDirectory() / "dozor_all")
os.system('touch {0}'.format(imageQualityIndicatorsDozorAllFile))
for controlDozorAllFile in listControlDozorAllFile:
command = 'cat ' + controlDozorAllFile + ' >> ' + imageQualityIndicatorsDozorAllFile
os.system(command)
outData["dozorAllFile"] = imageQualityIndicatorsDozorAllFile
outData['imageQualityIndicators'] = listImageQualityIndicators
outData['crystfel_results'] = listcrystfel_output
return outData
