def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
# Write workflowStepReport HTML page
pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
# Store in Pyarch
if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL():
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
if not os.path.exists(strPyarchPath):
os.makedirs(strPyarchPath, 0o755)
shutil.copy(pathToJsonFile, strPyarchPath)
pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
# Write json file
xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
self.setDataOutput(xsDataResultSimpleHTMLPage)
def __init__ (self):
EDPluginControl.__init__(self)
self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1"
self.strPluginName = "EDPluginDistlSignalStrengthv1_1"
self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1"
self.strPluginNameControlDozor = "EDPluginControlDozorv1_0"
self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_5"
self.strIndexingLabelitPluginName = "EDPluginLabelitIndexingv1_1"
self.edPluginLabelitIndexing = None
self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10"
self.edPluginReadImageHeader = None
self.edPluginControlDozor = None
self.setXSDataInputClass(XSDataInputControlImageQualityIndicators)
self.listPluginExecImageQualityIndicator = []
self.listPluginControlDozor = []
self.xsDataResultControlImageQualityIndicators = None
self.edPluginMXWaitFile = None
# Default time out for wait file
self.fMXWaitFileTimeOut = 30 # s
# Flag for using the thin client
if EDUtilsPath.isEMBL():
self.bUseThinClient = False
else:
self.bUseThinClient = True
self.edPluginISPyB = None
self.listPluginLabelit = []
self.defaultMinImageSize = 1000000
self.minImageSize = None
def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
# Write workflowStepReport HTML page
pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
# Store in Pyarch
if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV() or EDUtilsPath.isALBA:
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
if not os.path.exists(strPyarchPath):
os.makedirs(strPyarchPath, 0o755)
shutil.copy(pathToJsonFile, strPyarchPath)
pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
# Write json file
xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
self.setDataOutput(xsDataResultSimpleHTMLPage)
def configure(self):
EDPluginExecProcessScript.configure(self)
self.DEBUG("EDPluginExecXia2DIALSv1_0.configure")
self.maxNoProcessors = self.config.get("maxNoProcessors", self.maxNoProcessors)
if EDUtilsPath.isEMBL():
self.maxNoProcessors = multiprocessing.cpu_count() / 2
self.goniometerAxes = self.config.get("goniometerAxes", self.goniometerAxes)
def createPyarchFilePath(_strESRFPath):
"""
This method translates from an ESRF "visitor" path to a "pyarch" path:
/data/visitor/mx415/id14eh1/20100209 -> /data/pyarch/2010/id14eh1/mx415/20100209
"""
strPyarchDNAFilePath = None
listOfDirectories = _strESRFPath.split(os.sep)
if EDUtilsPath.isEMBL():
if 'p13' in listOfDirectories[0:3] or 'P13' in listOfDirectories[0:3]:
strPyarchDNAFilePath = os.path.join('/data/ispyb/p13',
*listOfDirectories[4:])
else:
strPyarchDNAFilePath = os.path.join('/data/ispyb/p14',
*listOfDirectories[4:])
return strPyarchDNAFilePath
listBeamlines = ["bm30a", "id14eh1", "id14eh2", "id14eh3", "id14eh4", "id23eh1", "id23eh2",
"id29", "id30a1", "id30a2", "id30a3", "id30b", "simulator_mxcube"]
# Check that we have at least four levels of directories:
if (len(listOfDirectories) > 5):
strDataDirectory = listOfDirectories[ 1 ]
strSecondDirectory = listOfDirectories[ 2 ]
strThirdDirectory = listOfDirectories[ 3 ]
strFourthDirectory = listOfDirectories[ 4 ]
strFifthDirectory = listOfDirectories[ 5 ]
year = strFifthDirectory[0:4]
strProposal = None
strBeamline = None
if (strDataDirectory == "data") and (strSecondDirectory == "gz"):
if strThirdDirectory == "visitor":
strProposal = strFourthDirectory
strBeamline = strFifthDirectory
elif strFourthDirectory == "inhouse":
strProposal = strFifthDirectory
strBeamline = strThirdDirectory
else:
raise RuntimeError("Illegal path for EDHandlerESRFPyarchv1_0.createPyarchFilePath: {0}".format(_strESRFPath))
listOfRemainingDirectories = listOfDirectories[ 6: ]
elif (strDataDirectory == "data") and (strSecondDirectory == "visitor"):
strProposal = listOfDirectories[ 3 ]
strBeamline = listOfDirectories[ 4 ]
listOfRemainingDirectories = listOfDirectories[ 5: ]
elif ((strDataDirectory == "data") and (strSecondDirectory in listBeamlines)):
strBeamline = strSecondDirectory
strProposal = listOfDirectories[ 4 ]
listOfRemainingDirectories = listOfDirectories[ 5: ]
if (strProposal != None) and (strBeamline != None):
strPyarchDNAFilePath = os.path.join(os.sep, "data")
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, "pyarch")
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, year)
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strBeamline)
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strProposal)
for strDirectory in listOfRemainingDirectories:
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strDirectory)
if (strPyarchDNAFilePath is None):
EDVerbose.WARNING("EDHandlerESRFPyarchv1_0.createPyArchFilePath: path not converted for pyarch: %s " % _strESRFPath)
return strPyarchDNAFilePath
def configure(self):
EDPluginExecProcessScript.configure(self)
self.DEBUG("EDPluginExecXia2DIALSv1_0.configure")
self.maxNoProcessors = self.config.get("maxNoProcessors",
self.maxNoProcessors)
if EDUtilsPath.isEMBL():
self.maxNoProcessors = multiprocessing.cpu_count() / 2
self.goniometerAxes = self.config.get("goniometerAxes",
self.goniometerAxes)
def postProcess(self, _edObject=None):
EDPluginExecProcessScript.postProcess(self)
self.DEBUG("EDPluginExecXia2DIALSv1_0.postProcess")
# Run xia2.ispyb_xml
currentDir = os.getcwd()
os.chdir(self.getWorkingDirectory())
if EDUtilsPath.isEMBL():
subprocess.call("/mx-beta/dials-v1-5-1/build/bin/xia2.ispyb_xml")
else:
subprocess.call("/opt/pxsoft/bin/xia2.ispyb_xml")
os.chdir(currentDir)
# Populate the results
xsDataResultXia2DIALS = self.parseOutputDirectory(self.getWorkingDirectory())
self.dataOutput = xsDataResultXia2DIALS
def preProcess(self):
EDPluginExecProcessScript.preProcess(self)
self.DEBUG('Pointless: preprocess')
if self.output_file is not None and self.input_file is not None:
if EDUtilsPath.isEMBL():
options = '''-c xdsin {0} hklout {1}'''.format(self.input_file,
self.output_file)
else:
options = '''xdsin {0} hklout {1}'''.format(self.input_file,
self.output_file)
self.setScriptCommandline(options)
self.DEBUG('command line options set to {0}'.format(options))
self.addListCommandExecution('setting symmetry-based')
if self.dataInput.choose_spacegroup is not None:
self.addListCommandExecution('choose spacegroup {0}'.format(self.dataInput.choose_spacegroup.value))
def postProcess(self, _edObject=None):
EDPluginExecProcessScript.postProcess(self)
self.DEBUG("EDPluginExecXia2DIALSv1_0.postProcess")
# Run xia2.ispyb_xml
currentDir = os.getcwd()
os.chdir(self.getWorkingDirectory())
if EDUtilsPath.isEMBL():
subprocess.call("/mx-beta/dials-v1-5-1/build/bin/xia2.ispyb_xml")
else:
subprocess.call("/opt/pxsoft/bin/xia2.ispyb_xml")
os.chdir(currentDir)
# Populate the results
xsDataResultXia2DIALS = self.parseOutputDirectory(
self.getWorkingDirectory())
self.dataOutput = xsDataResultXia2DIALS
def preProcess(self):
EDPluginExecProcessScript.preProcess(self)
self.DEBUG('Pointless: preprocess')
if self.output_file is not None and self.input_file is not None:
if EDUtilsPath.isEMBL() or EDUtilsPath.isALBA():
options = '''-c xdsin {0} hklout {1}'''.format(
self.input_file, self.output_file)
else:
options = '''xdsin {0} hklout {1}'''.format(
self.input_file, self.output_file)
self.setScriptCommandline(options)
self.DEBUG('command line options set to {0}'.format(options))
self.addListCommandExecution('setting symmetry-based')
if self.dataInput.choose_spacegroup is not None:
self.addListCommandExecution('choose spacegroup {0}'.format(
self.dataInput.choose_spacegroup.value))
def generateCommandLine(self, _xsDataInputXia2DIALS):
"""
This method creates the input command line for Xia2DIALS
"""
self.DEBUG("EDPluginExecXia2DIALSv1_0.generateCommands")
if EDUtilsPath.isEMBL():
strCommandText = "pipeline=dials ispyb_xml_out ispyb.xml"
else:
strCommandText = "pipeline=dials"
anomalous = True
if _xsDataInputXia2DIALS.anomalous is not None:
if not _xsDataInputXia2DIALS.anomalous.value:
anomalous = False
if anomalous:
strCommandText += " atom=X"
if _xsDataInputXia2DIALS.startFrame is not None and _xsDataInputXia2DIALS.endFrame is not None:
startFrame = _xsDataInputXia2DIALS.startFrame.value
endFrame = _xsDataInputXia2DIALS.endFrame.value
for image in _xsDataInputXia2DIALS.image:
strCommandText += " image={0}:{1}:{2}".format(
image.path.value, startFrame, endFrame)
else:
for image in _xsDataInputXia2DIALS.image:
strCommandText += " image={0}".format(image.path.value)
if self.maxNoProcessors is not None:
strCommandText += " multiprocessing.nproc={0}".format(
self.maxNoProcessors)
if self.goniometerAxes is not None:
strCommandText += " goniometer.axes={0}".format(
self.goniometerAxes)
if _xsDataInputXia2DIALS.spaceGroup is not None:
strCommandText += " xia2.settings.space_group={0}".format(
_xsDataInputXia2DIALS.spaceGroup.value)
if _xsDataInputXia2DIALS.unitCell is not None:
unitCell = _xsDataInputXia2DIALS.unitCell.value
if not "," in unitCell:
unitCell.replace(" ", ",")
strCommandText += " xia2.settings.unit_cell={0}".format(unitCell)
return strCommandText
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlRunXdsFastProcv1_0.preProcess")
# Load the execution plugin
self.first_run = self.loadPlugin(self.controlled_plugin_name)
cfg = parse_xds_file(self.dataInput.input_file.value)
spot_range = cfg.get('SPOT_RANGE=')
if spot_range is None:
if EDUtilsPath.isEMBL():
# GB: add a default here - it is nothing worthe then the
# one in autogenerated XDS.INP
pass
else:
strErrorMessage = "No SPOT_RANGE parameter"
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
else:
self.spot_range = spot_range
# we will use this value to constrain the upper bound of
# spot_range so it does not get past the last image number, so
# we use a default value that cannot be a constraint in case
# we cannot find it in the xds input file
self.end_image_no = sys.maxsize
data_range = cfg.get('DATA_RANGE=')
if data_range is not None:
self.start_image_no = data_range[0]
self.end_image_no = data_range[1]
# GB: repair the above.... but why spot_range should be a list of lists?
if spot_range is None:
self.spot_range = [[
max(1, data_range[0]),
min(data_range[0] + 99, data_range[1])
]]
else:
# Check that spot ranges is within data range
new_spot_range = []
for spot_range in self.spot_range:
if spot_range[0] >= self.start_image_no and spot_range[
1] <= self.end_image_no:
new_spot_range.append(spot_range)
self.spot_range = new_spot_range
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlRunXdsFastProcv1_0.preProcess")
# Load the execution plugin
self.first_run = self.loadPlugin(self.controlled_plugin_name)
cfg = parse_xds_file(self.dataInput.input_file.value)
spot_range = cfg.get('SPOT_RANGE=')
if spot_range is None:
if EDUtilsPath.isEMBL():
# GB: add a default here - it is nothing worthe then the
# one in autogenerated XDS.INP
pass
else:
strErrorMessage = "No SPOT_RANGE parameter"
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
else:
self.spot_range = spot_range
# we will use this value to constrain the upper bound of
# spot_range so it does not get past the last image number, so
# we use a default value that cannot be a constraint in case
# we cannot find it in the xds input file
self.end_image_no = sys.maxsize
data_range = cfg.get('DATA_RANGE=')
if data_range is not None:
self.start_image_no = data_range[0]
self.end_image_no = data_range[1]
# GB: repair the above.... but why spot_range should be a list of lists?
if spot_range is None:
self.spot_range = [[max(1, data_range[0]),
min(data_range[0] + 99, data_range[1])]]
else:
# Check that spot ranges is within data range
new_spot_range = []
for spot_range in self.spot_range:
if spot_range[0] >= self.start_image_no and spot_range[1] <= self.end_image_no:
new_spot_range.append(spot_range)
self.spot_range = new_spot_range
def generateCommandLine(self, _xsDataInputXia2DIALS):
"""
This method creates the input command line for Xia2DIALS
"""
self.DEBUG("EDPluginExecXia2DIALSv1_0.generateCommands")
if EDUtilsPath.isEMBL():
strCommandText = "pipeline=dials ispyb_xml_out ispyb.xml"
else:
strCommandText = "pipeline=dials"
anomalous = True
if _xsDataInputXia2DIALS.anomalous is not None:
if not _xsDataInputXia2DIALS.anomalous.value:
anomalous = False
if anomalous:
strCommandText += " atom=X"
if _xsDataInputXia2DIALS.startFrame is not None and _xsDataInputXia2DIALS.endFrame is not None:
startFrame = _xsDataInputXia2DIALS.startFrame.value
endFrame = _xsDataInputXia2DIALS.endFrame.value
for image in _xsDataInputXia2DIALS.image:
strCommandText += " image={0}:{1}:{2}".format(image.path.value, startFrame, endFrame)
else:
for image in _xsDataInputXia2DIALS.image:
strCommandText += " image={0}".format(image.path.value)
if self.maxNoProcessors is not None:
strCommandText += " multiprocessing.nproc={0}".format(self.maxNoProcessors)
if self.goniometerAxes is not None:
strCommandText += " goniometer.axes={0}".format(self.goniometerAxes)
if _xsDataInputXia2DIALS.spaceGroup is not None:
strCommandText += " xia2.settings.space_group={0}".format(_xsDataInputXia2DIALS.spaceGroup.value)
if _xsDataInputXia2DIALS.unitCell is not None:
unitCell = _xsDataInputXia2DIALS.unitCell.value
if not "," in unitCell:
unitCell.replace(" ", ",")
strCommandText += " xia2.settings.unit_cell={0}".format(unitCell)
return strCommandText
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
if EDUtilsPath.isEMBL():
self.__fDefaultTimeOutInSeconds = 3600.0
else:
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
if EDUtilsPath.isEMBL() or EDUtilsPath.isALBA():
self.__fDefaultTimeOutInSeconds = 3600.0
else:
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [
] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
strImageFileNameExtension = os.path.splitext(
strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [
".img", ".marccd", ".mccd", ".cbf", ".h5"
]:
self.error(
"Unknown image file name extension for pyarch thumbnail generator: %s"
% strPathToDiffractionImage)
self.setFailure()
else:
# Load the MXWaitFile plugin
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
pathToImageFile = strPathToDiffractionImage
# Quite ugly hack to avoid lag problems at the ESRF:
if EDUtilsPath.isESRF() or EDUtilsPath.isALBA():
if any(beamline in strPathToDiffractionImage
for beamline in ["id23eh1", "id29", "id30b"]):
# Pilatus 6M
self.minImageSize = 6000000
elif any(beamline in strPathToDiffractionImage
for beamline in ["id23eh2", "id30a1"]):
# Pilatus3 2M
self.minImageSize = 2000000
elif strImageFileNameExtension == ".h5":
self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(
pathToImageFile)
pathToImageFile = self.h5DataFilePath
self.isH5 = True
elif EDUtilsPath.isEMBL():
self.minImageSize = 10000
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setFile(
XSDataFile(XSDataString(pathToImageFile)))
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputMXWaitFile.setTimeOut(
self.getDataInput().getWaitForFileTimeOut())
self.edPluginMXWaitFile = self.loadPlugin(
self.strMXWaitFilePluginName)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(
self.strExecThumbnailPluginName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = self.getDataInput(
).getDiffractionImage()
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
xsDataInputMXThumbnail.format = self.dataInput.format
# Output path
strImageNameWithoutExt = os.path.basename(
os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput(
).getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
self.error("Cannot write to forced output directory : %s" %
strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(
strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except Exception as e:
self.WARNING("Couldn't create the directory %s" %
strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
self.warning("Cannot write to pyarch directory: %s" %
strOutputDirname)
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0o755)
strOutputDirname = tempfile.mkdtemp(
prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
os.chmod(strOutputDirname, 0o755)
self.warning("Writing thumbnail images to: %s" %
strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(
strOutputDirname, strImageNameWithoutExt)
if self.dataInput.format is not None:
self.strSuffix = self.dataInput.format.value.lower()
self.strImageFormat = self.dataInput.format.value.upper()
self.strOutputPath = os.path.join(
self.strOutputPathWithoutExtension + "." + self.strSuffix)
xsDataInputMXThumbnail.setOutputPath(
XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlCrystFELv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
userName = os.environ["USER"]
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
#self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
# "CrystFEL",
# "started")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
self.ispyb_retrieve_dc_plugin.executeSynchronous()
ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
#TODO PE2 has 7 digits others 5
template = ispybDataCollection.fileTemplate.replace(
"%07d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
dir_parts = processDirectory.split("/")
processDirectory = "/data/users/%s" % os.path.join(*dir_parts[5:])
# Make results directory
self.resultsDirectory = os.path.join(processDirectory,
"crystfel_results")
try:
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory)
except Exception as ex:
print ex
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
directory)
#TODO: enable when ready
"""
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory)
"""
self.baseFileName = "%s/crystfel_xgandalf_%d" % (
self.resultsDirectory, self.dataInput.dataCollectionId.value)
xsDataInputCrystFEL = XSDataInputCrystFEL()
# Check if imagesFullPath exists. If not then generate one
if self.dataInput.imagesFullPath is None:
imagesFullPath = "%s_images_fullpath.lst" % self.baseName
imagesFullFile = open(imagesFullPath, "w")
for index in range(imageNoEnd - imageNoStart + 1):
imagePath = os.path.join(
directory, ispybDataCollection.fileTemplate % (index + 1))
imagesFullFile.write("%s\n" % imagePath)
imagesFullFile.close()
xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
else:
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
xsDataInputCrystFEL.pointGroup = self.dataInput.pointGroup
xsDataInputCrystFEL.spaceGroup = self.dataInput.spaceGroup
xsDataInputCrystFEL.resCutOff = self.dataInput.resCutOff
xsDataInputCrystFEL.baseFileName = XSDataString(self.baseFileName)
self.timeStart = time.localtime()
self.integration_id, self.program_id = self.create_integration_id(
self.dataInput.dataCollectionId.value, "Creating integration ID")
self.index_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
self.partialator_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataOutput = XSDataResultCrystFEL
self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
self.process_hkl_plugin_odd.process_hkl_type = "o"
self.process_hkl_plugin_even.process_hkl_options += " --even-only"
self.process_hkl_plugin_even.process_hkl_type = "e"
self.index_plugin.executeSynchronous()
if self.index_plugin.isFailure():
self.ERROR('indexamajig: Failed')
self.setFailure()
return
else:
self.screen('indexamajig: Finished')
self.process_hkl_plugin.executeSynchronous()
if self.process_hkl_plugin.isFailure():
self.ERROR('process_hkl: Failed')
self.setFailure()
return
else:
self.screen('process_hkl: Finished')
self.process_hkl_plugin_odd.executeSynchronous()
if self.process_hkl_plugin_odd.isFailure():
self.ERROR('process_hkl_odd: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_odd: Finished')
self.process_hkl_plugin_even.executeSynchronous()
if self.process_hkl_plugin_even.isFailure():
self.ERROR('process_hkl_even: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_even: Finished')
self.partialator_plugin.executeSynchronous()
if self.partialator_plugin.isFailure():
self.ERROR('partialator: Failed')
self.setFailure()
return
else:
self.screen('partialator: Finished')
self.post_process_plugin.executeSynchronous()
if self.post_process_plugin.isFailure():
self.ERROR('post_process: Failed')
self.setFailure()
return
else:
self.screen('post_process: Finished')
self.timeEnd = time.localtime()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doNoanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
if self.doAnom:
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
else:
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doAnomAndNonanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doAnomAndNonanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def process(self):
self.DEBUG('FileConversion: process')
EDPluginControl.process(self)
# first we generate the intermediary file name
pointless_in = XSDataPointless()
pointless_in.input_file = self.dataInput.input_file
pointless_out = os.path.join(os.path.dirname(self.dataInput.output_file.value),
self.pointless_out)
pointless_in.output_file = XSDataString(pointless_out)
if self.dataInput.choose_spacegroup is not None:
pointless_in.choose_spacegroup = self.dataInput.choose_spacegroup
self.pointless.dataInput = pointless_in
self.screen("Pointless run " + self.strAnomSuffix)
self.pointless.executeSynchronous()
if self.pointless.isFailure():
strErrorMessage = "Pointless {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
aimless_in = XSDataAimless()
aimless_in.input_file = pointless_in.output_file
aimless_in.output_file = XSDataString(os.path.join(self.results_dir,
self.aimless_out))
aimless_in.command_file = XSDataString(os.path.join(self.results_dir,
self.aimless_commands_out))
aimless_in.dataCollectionID = self.dataInput.dataCollectionID
aimless_in.start_image = self.dataInput.start_image
aimless_in.end_image = self.dataInput.end_image
aimless_in.res = self.dataInput.res
aimless_in.anom = self.dataInput.anom
self.aimless.dataInput = aimless_in
self.screen("Aimless run " + self.strAnomSuffix)
self.aimless.executeSynchronous()
if self.aimless.isFailure():
strErrorMessage = "Aimless {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# copy the aimless log where the results files are
source_log = os.path.join(self.aimless.getWorkingDirectory(),
self.aimless.getScriptLogFileName())
self.aimless_log = source_log
target_log = os.path.join(self.results_dir,
'ep_{0}aimless_{1}.log'.format(self.image_prefix,
"anom" if self.dataInput.anom.value else "noanom"))
try:
shutil.copyfile(source_log, target_log)
except IOError:
self.ERROR('Could not copy aimless log file from {0} to {1}'.format(source_log,
target_log))
# now truncate
truncate_in = XSDataTruncate()
truncate_in.input_file = self.aimless.dataInput.output_file
temp_file = tempfile.NamedTemporaryFile(suffix='.mtz',
prefix='tmp2-',
dir=self.aimless.getWorkingDirectory(),
delete=False)
truncate_out = temp_file.name
temp_file.close()
os.chmod(truncate_out, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
truncate_in.output_file = XSDataString(truncate_out)
truncate_in.nres = self.dataInput.nres
truncate_in.anom = self.dataInput.anom
truncate_in.res = self.dataInput.res
self.truncate.dataInput = truncate_in
self.screen("Truncate run " + self.strAnomSuffix)
self.truncate.executeSynchronous()
if self.truncate.isFailure():
strErrorMessage = "Truncate {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# copy the truncate log where the results files are
source_log = os.path.join(self.truncate.getWorkingDirectory(),
self.truncate.getScriptLogFileName())
target_log = os.path.join(self.results_dir,
'ep_{0}truncate_{1}.log'.format(self.image_prefix,
"anom" if self.dataInput.anom.value else "noanom"))
try:
shutil.copyfile(source_log,
target_log)
except IOError:
self.ERROR('Could not copy truncate log file from {0} to {1}'.format(source_log,
target_log))
# and finally uniqueify
uniqueify_in = XSDataUniqueify()
uniqueify_in.input_file = truncate_in.output_file
uniqueify_out = os.path.join(self.results_dir,
self.truncate_out)
uniqueify_in.output_file = XSDataString(uniqueify_out)
self.uniqueify.dataInput = uniqueify_in
if EDUtilsPath.isEMBL():
#GB: skipping misteriously failing uniqueify run -
# which is useless anyway.
# copying temp truncate output to results directly
shutil.copyfile(uniqueify_in.input_file.value, uniqueify_out)
return
self.screen("Uniqueify run " + self.strAnomSuffix)
self.uniqueify.executeSynchronous()
if self.uniqueify.isFailure():
strErrorMessage = "Uniqueify {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
def generateCommandLine(self, _xsDataInputAutoPROC):
"""
This method creates the input command line for autoPROC
"""
self.DEBUG("EDPluginExecAutoPROCv1_0.generateCommands")
strCommandText = "-B -xml -nthreads {0} -M ReportingInlined autoPROC_HIGHLIGHT=\"no\"".format(self.maxNoProcessors)
if self.doScaleWithXscale:
strCommandText += " autoPROC_ScaleWithXscale='yes'"
if self.rotationAxis is not None:
strCommandText += " autoPROC_XdsKeyword_ROTATION_AXIS=\"{0}\"".format(self.rotationAxis)
# Master H5 file
masterH5 = _xsDataInputAutoPROC.masterH5
if masterH5 is None:
# Identifier(s)
for identifier in _xsDataInputAutoPROC.identifier:
if EDUtilsPath.isEMBL() or EDUtilsPath.isALBA():
identifier.templateN.value = identifier.templateN.value.replace(\
'%' + '05' + 'd', 5 * '#')
strCommandText += " -Id {idN},{dirN},{templateN},{fromN},{toN}".format(
idN=identifier.idN.value,
dirN=identifier.dirN.path.value,
templateN=identifier.templateN.value,
fromN=identifier.fromN.value,
toN=identifier.toN.value)
else:
strCommandText += " -h5 {0}".format(masterH5.path.value)
if self.pathToNeggiaPlugin is not None:
strCommandText += " autoPROC_XdsKeyword_LIB={0}".format(self.pathToNeggiaPlugin)
# Resolution
lowResolutionLimit = _xsDataInputAutoPROC.lowResolutionLimit
highResolutionLimit = _xsDataInputAutoPROC.highResolutionLimit
if lowResolutionLimit is not None or highResolutionLimit is not None:
# See https://www.globalphasing.com/autoproc/manual/autoPROC4.html#processcli
if lowResolutionLimit is None:
low = 1000.0 # autoPROC default value
else:
low = lowResolutionLimit.value
if highResolutionLimit is None:
high = 0.1 # autoPROC default value
else:
high = highResolutionLimit.value
strCommandText += " -R {0} {1}".format(low, high)
# Anomalous
anomalous = _xsDataInputAutoPROC.anomalous
if anomalous is not None:
if anomalous.value:
strCommandText += " -ANO"
# Reference MTZ file
refMTZ = _xsDataInputAutoPROC.refMTZ
if refMTZ is not None:
strCommandText += " -ref {0}".format(refMTZ.path.value)
# Forced space group
if _xsDataInputAutoPROC.symm is not None:
strCommandText += " symm=\"{0}\"".format(_xsDataInputAutoPROC.symm.value)
# Forced cell
if _xsDataInputAutoPROC.cell is not None:
strCommandText += " cell=\"{0}\"".format(_xsDataInputAutoPROC.cell.value)
return strCommandText
def process(self):
self.DEBUG('FileConversion: process')
EDPluginControl.process(self)
# first we generate the intermediary file name
pointless_in = XSDataPointless()
pointless_in.input_file = self.dataInput.input_file
pointless_out = os.path.join(os.path.dirname(self.dataInput.output_file.value),
self.pointless_out)
pointless_in.output_file = XSDataString(pointless_out)
if self.dataInput.choose_spacegroup is not None:
pointless_in.choose_spacegroup = self.dataInput.choose_spacegroup
self.pointless.dataInput = pointless_in
self.screen("Pointless run " + self.strAnomSuffix)
self.pointless.executeSynchronous()
if self.pointless.isFailure():
strErrorMessage = "Pointless {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
aimless_in = XSDataAimless()
aimless_in.input_file = pointless_in.output_file
aimless_in.output_file = XSDataString(os.path.join(self.results_dir,
self.aimless_out))
aimless_in.command_file = XSDataString(os.path.join(self.results_dir,
self.aimless_commands_out))
aimless_in.dataCollectionID = self.dataInput.dataCollectionID
aimless_in.start_image = self.dataInput.start_image
aimless_in.end_image = self.dataInput.end_image
aimless_in.res = self.dataInput.res
aimless_in.anom = self.dataInput.anom
self.aimless.dataInput = aimless_in
self.screen("Aimless run " + self.strAnomSuffix)
self.aimless.executeSynchronous()
if self.aimless.isFailure():
strErrorMessage = "Aimless {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# copy the aimless log where the results files are
source_log = os.path.join(self.aimless.getWorkingDirectory(),
self.aimless.getScriptLogFileName())
self.aimless_log = source_log
target_log = os.path.join(self.results_dir,
'ep_{0}aimless_{1}.log'.format(self.image_prefix,
"anom" if self.dataInput.anom.value else "noanom"))
try:
shutil.copyfile(source_log, target_log)
except IOError:
self.ERROR('Could not copy aimless log file from {0} to {1}'.format(source_log,
target_log))
# now truncate
truncate_in = XSDataTruncate()
truncate_in.input_file = self.aimless.dataInput.output_file
temp_file = tempfile.NamedTemporaryFile(suffix='.mtz',
prefix='tmp2-',
dir=self.aimless.getWorkingDirectory(),
delete=False)
truncate_out = temp_file.name
temp_file.close()
os.chmod(truncate_out, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
truncate_in.output_file = XSDataString(truncate_out)
truncate_in.nres = self.dataInput.nres
truncate_in.anom = self.dataInput.anom
truncate_in.res = self.dataInput.res
self.truncate.dataInput = truncate_in
self.screen("Truncate run " + self.strAnomSuffix)
self.truncate.executeSynchronous()
if self.truncate.isFailure():
strErrorMessage = "Truncate {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
# copy the truncate log where the results files are
source_log = os.path.join(self.truncate.getWorkingDirectory(),
self.truncate.getScriptLogFileName())
target_log = os.path.join(self.results_dir,
'ep_{0}truncate_{1}.log'.format(self.image_prefix,
"anom" if self.dataInput.anom.value else "noanom"))
try:
shutil.copyfile(source_log,
target_log)
except IOError:
self.ERROR('Could not copy truncate log file from {0} to {1}'.format(source_log,
target_log))
# and finally uniqueify
uniqueify_in = XSDataUniqueify()
uniqueify_in.input_file = truncate_in.output_file
uniqueify_out = os.path.join(self.results_dir,
self.truncate_out)
uniqueify_in.output_file = XSDataString(uniqueify_out)
self.uniqueify.dataInput = uniqueify_in
if EDUtilsPath.isEMBL() or EDUtilsPath.isALBA():
# GB: skipping misteriously failing uniqueify run -
# which is useless anyway.
# copying temp truncate output to results directly
shutil.copyfile(uniqueify_in.input_file.value, uniqueify_out)
return
self.screen("Uniqueify run " + self.strAnomSuffix)
self.uniqueify.executeSynchronous()
if self.uniqueify.isFailure():
strErrorMessage = "Uniqueify {0} failed".format(self.strAnomSuffix)
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
return
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlDozorv1_0.process")
xsDataResultControlDozor = XSDataResultControlDozor()
# Check if connection to ISPyB needed
if self.dataInput.dataCollectionId is not None:
edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
batchSize = ispybDataCollection.numberOfImages
if batchSize > self.maxBatchSize:
batchSize = self.maxBatchSize
if abs(ispybDataCollection.overlap) > 1:
self.hasOverlap = True
self.overlap = ispybDataCollection.overlap
dictImage = self.createImageDictFromISPyB(ispybDataCollection)
else:
# No connection to ISPyB, take parameters from input
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
dictImage = self.createImageDict(self.dataInput)
listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
# Convert HDF5 images to CBF
self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage)
for listBatch in listAllBatches:
# Read the header from the first image in the batch
xsDataFile = dictImage[listBatch[0]]
edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = xsDataFile
edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
edPluginControlReadImageHeader.executeSynchronous()
subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
xsDataInputDozor = XSDataInputDozor()
beam = subWedge.experimentalCondition.beam
detector = subWedge.experimentalCondition.detector
goniostat = subWedge.experimentalCondition.goniostat
xsDataInputDozor.detectorType = detector.type
xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
xsDataInputDozor.spotSize = XSDataInteger(3)
xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
# xsDataInputDozor.fractionPolatization : XSDataDouble optional
orgx = detector.beamPositionY.value / detector.pixelSizeY.value
orgy = detector.beamPositionX.value / detector.pixelSizeX.value
xsDataInputDozor.orgx = XSDataDouble(orgx)
xsDataInputDozor.orgy = XSDataDouble(orgy)
xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
# xsDataInputDozor.imageStep : XSDataDouble optional
xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
xsDataInputDozor.firstImageNumber = subWedge.image[0].number
xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
if self.hasOverlap:
xsDataInputDozor.overlap = XSDataAngle(self.overlap)
strFileName = subWedge.image[0].path.value
strPrefix = EDUtilsImage.getPrefix(strFileName)
strSuffix = EDUtilsImage.getSuffix(strFileName)
if EDUtilsPath.isEMBL():
strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
elif self.hasHdf5Prefix and not self.hasOverlap:
strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
else:
strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
xsDataInputDozor.nameTemplateImage = XSDataString(
os.path.join(os.path.dirname(strFileName), strXDSTemplate)
)
xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
edPluginDozor.dataInput = xsDataInputDozor
edPluginDozor.executeSynchronous()
indexImage = 0
for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
xsDataControlImageDozor = XSDataControlImageDozor()
xsDataControlImageDozor.number = xsDataResultDozor.number
xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
xsDataControlImageDozor.angle = xsDataResultDozor.angle
xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
if xsDataResultControlDozor.inputDozor is None:
xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
indexImage += 1
xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
self.dataOutput = xsDataResultControlDozor
if self.cbfTempDir is not None:
shutil.rmtree(self.cbfTempDir)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlCrystFELv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
#self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
# "CrystFEL",
# "started")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
self.ispyb_retrieve_dc_plugin.executeSynchronous()
ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
#TODO PE2 has 7 digits others 5
template = ispybDataCollection.fileTemplate.replace("%07d", "####")
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
try:
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 755)
except Exception as ex:
print ex
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
#TODO: enable when ready
"""
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 755)
"""
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 60
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst)
self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst
self.wait_first_image_plugin.executeSynchronous()
if self.wait_first_image_plugin.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory
self.streamFilename = "%s_%d.stream" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.hklFilename = "%s_%d.hkl" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.mtzFilename = "%s_%d.mtz" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
# Prepare input to execution plugin
xsDataInputCrystFEL = XSDataInputCrystFEL()
# Check if imagesFullPath exists. If not then generate one
if self.dataInput.imagesFullPath is None:
imagesFullPath = "%s_images_fullpath.lst" % self.baseName
imagesFullFile = open(imagesFullPath, "w")
for index in range(imageNoEnd - imageNoStart + 1):
imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1))
imagesFullFile.write("%s\n" % imagePath)
imagesFullFile.close()
xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
else:
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
#TODO EDNA can generate the imagesFull path if id do not exist
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename)
xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename)
xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename)
self.timeStart = time.localtime()
"""
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationId, self.autoProcProgramId = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
"""
self.index_plugin.dataInput = xsDataInputCrystFEL
self.index_plugin.executeSynchronous()
if self.index_plugin.isFailure():
self.ERROR('indexamajig: Failed')
self.setFailure()
return
else:
self.screen('indexamajig: Finished')
self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
self.partialator_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
self.process_hkl_plugin_even.process_hkl_options += " --even-only"
self.process_hkl_plugin.executeSynchronous()
if self.process_hkl_plugin.isFailure():
self.ERROR('process_hkl: Failed')
self.setFailure()
return
else:
self.screen('process_hkl: Finished')
self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o")
self.process_hkl_plugin_odd.executeSynchronous()
if self.process_hkl_plugin_odd.isFailure():
self.ERROR('process_hkl_odd: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_odd: Finished')
self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e")
self.process_hkl_plugin_even.executeSynchronous()
if self.process_hkl_plugin_even.isFailure():
self.ERROR('process_hkl_even: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_even: Finished')
self.partialator_plugin.executeSynchronous()
if self.partialator_plugin.isFailure():
self.ERROR('partialator: Failed')
self.setFailure()
return
else:
self.screen('partialator: Finished')
self.post_process_plugin.executeSynchronous()
if self.post_process_plugin.isFailure():
self.ERROR('post_process: Failed')
self.setFailure()
return
else:
self.screen('post_process: Finished')
"""
self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL
self.edPluginExecCrystFELScale.executeSynchronous()
if self.edPluginExecCrystFELScale.isFailure():
self.ERROR('CrystFELScalev1_0: Failed')
self.setFailure()
return
else:
self.screen('CrystFELScalev1_0: Finished')
self.timeEnd = time.localtime()
"""
# Upload to ISPyB
"""
def preProcess(self, _edObject=None):
EDPluginExecProcessScript.preProcess(self)
self.DEBUG("EDPluginMinimalXDS.preProcess")
xds_input = os.path.abspath(self.dataInput.input_file.value)
workingDirectory = self.getWorkingDirectory()
if not os.path.exists(workingDirectory):
os.makedirs(workingDirectory, 0o755)
shutil.copy(xds_input, workingDirectory)
# our new xds file
xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
parsed_config = parse_xds_file(xds_file)
file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
# Find out the real path for the template
file_template_real_path = os.path.join(os.path.dirname(xds_input),
file_template)
# get the real directory the files are in by getting the real
# path of the first image
first_image_no = parsed_config['DATA_RANGE='][0]
first_image = _template_to_image(file_template_real_path,
first_image_no)
parsed_config[
'NAME_TEMPLATE_OF_DATA_FRAMES='] = file_template_real_path
# perhaps modify some params
job = self.dataInput.job
maxproc = self.dataInput.maxproc
maxjobs = self.dataInput.maxjobs
resolution_range = self.dataInput.resolution_range
friedels_law = self.dataInput.friedels_law
spot_range = self.dataInput.spot_range
spacegroup = self.dataInput.spacegroup
unit_cell = self.dataInput.unit_cell
self.DEBUG('requested spot range is {0}'.format(spot_range))
if job is not None:
parsed_config["JOB="] = job.value
if maxproc is not None:
parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
if maxjobs is not None:
parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
if resolution_range is not None and len(resolution_range) != 0:
parsed_config["INCLUDE_RESOLUTION_RANGE="] = [
x.value for x in resolution_range
]
if friedels_law is not None:
if friedels_law.value:
parsed_config["FRIEDEL'S_LAW="] = "TRUE"
else:
parsed_config["FRIEDEL'S_LAW="] = "FALSE"
if self.dataInput.start_image is not None:
start_image = self.dataInput.start_image.value
parsed_config['DATA_RANGE='][0] = start_image
else:
start_image = parsed_config['DATA_RANGE='][0]
if self.dataInput.end_image is not None:
end_image = self.dataInput.end_image.value
parsed_config['DATA_RANGE='][1] = end_image
else:
end_image = parsed_config['DATA_RANGE='][1]
spot_range_list = list()
if spot_range is not None and len(spot_range) > 0:
spot_range_list = []
for srange in spot_range:
spot_range_list.append([srange.begin, srange.end])
spot_range_list = self.checkSpotRanges(spot_range_list,
start_image, end_image)
elif 'SPOT_RANGE=' in parsed_config:
spot_range_list = self.checkSpotRanges(
parsed_config['SPOT_RANGE='], start_image, end_image)
else:
spot_range_list = [[start_image, end_image]]
self.DEBUG('setting the spot range to {0}'.format(spot_range_list))
parsed_config['SPOT_RANGE='] = spot_range_list
# Check background range
if 'BACKGROUND_RANGE=' in parsed_config:
background_range = parsed_config['BACKGROUND_RANGE=']
if background_range[0] < start_image:
background_range[0] = start_image
background_range[1] += start_image - 1
if background_range[1] > end_image:
background_range[1] = end_image
parsed_config['BACKGROUND_RANGE='] = background_range
# unit cell might be an empty string or some other crazy stuff
# we need 6 floats/ints
if unit_cell is not None:
ucells = unit_cell.value.split(' ')
if len(ucells) != 6:
unit_cell = None
else:
try:
if any(float(x) == 0 for x in ucells):
unit_cell = None
except ValueError:
unit_cell = None
# both need to be specified
if spacegroup is not None and unit_cell is not None:
self.DEBUG('specific spacegroup requested: {0}'.format(
spacegroup.value))
self.DEBUG('specific unit cell requested: {0}'.format(
unit_cell.value))
parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
# Check if this is ok
parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value
# For [XY]-GEO_CORR files, link them in the cwd and fix their paths
if 'X-GEO_CORR=' in parsed_config:
xgeo = parsed_config['X-GEO_CORR='][0]
xgeo_path = os.path.join(self.getWorkingDirectory(),
os.path.basename(xgeo))
if not os.path.exists(xgeo_path):
os.symlink(xgeo, xgeo_path)
parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
if 'Y-GEO_CORR=' in parsed_config:
ygeo = parsed_config['Y-GEO_CORR='][0]
ygeo_path = os.path.join(self.getWorkingDirectory(),
os.path.basename(ygeo))
if not os.path.exists(ygeo_path):
os.symlink(ygeo, ygeo_path)
parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)
# Neggia plugin
if not "LIB=" in parsed_config and self.pathToNeggiaPlugin is not None:
parsed_config["LIB="] = self.pathToNeggiaPlugin
# Max no processors
if not EDUtilsPath.isEMBL():
parsed_config['MAXIMUM_NUMBER_OF_PROCESSORS='] = str(
self.maxNoProcessors)
parsed_config['MAXIMUM_NUMBER_OF_JOBS='] = 1
# Save back the changes
dump_xds_file(xds_file, parsed_config)
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]:
self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
self.setFailure()
else:
# Load the MXWaitFile plugin
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
pathToImageFile = strPathToDiffractionImage
# Quite ugly hack to avoid lag problems at the ESRF:
if EDUtilsPath.isESRF():
if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]):
# Pilatus 6M
self.minImageSize = 6000000
elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]):
# Pilatus3 2M
self.minImageSize = 2000000
elif strImageFileNameExtension == ".h5":
self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile)
pathToImageFile = self.h5DataFilePath
self.isH5 = True
elif EDUtilsPath.isEMBL():
self.minImageSize = 10000
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile)))
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage()
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
xsDataInputMXThumbnail.format = self.dataInput.format
# Output path
strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except Exception as e:
self.WARNING("Couldn't create the directory %s" % strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
self.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0755)
strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
os.chmod(strOutputDirname, 0755)
self.warning("Writing thumbnail images to: %s" % strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
if self.dataInput.format is not None:
self.strSuffix = self.dataInput.format.value.lower()
self.strImageFormat = self.dataInput.format.value.upper()
self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix)
xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def createPyarchFilePath(_strESRFPath):
"""
This method translates from an ESRF "visitor" path to a "pyarch" path:
/data/visitor/mx415/id14eh1/20100209 -> /data/pyarch/2010/id14eh1/mx415/20100209
"""
strPyarchDNAFilePath = None
listOfDirectories = _strESRFPath.split(os.sep)
if EDUtilsPath.isEMBL():
if 'p13' in listOfDirectories[0:3] or 'P13' in listOfDirectories[
0:3]:
strPyarchDNAFilePath = os.path.join('/data/ispyb/p13',
*listOfDirectories[4:])
else:
strPyarchDNAFilePath = os.path.join('/data/ispyb/p14',
*listOfDirectories[4:])
return strPyarchDNAFilePath
listBeamlines = [
"bm30a", "id14eh1", "id14eh2", "id14eh3", "id14eh4", "id23eh1",
"id23eh2", "id29", "id30a1", "id30a2", "id30a3", "id30b",
"simulator_mxcube"
]
# Check that we have at least four levels of directories:
if (len(listOfDirectories) > 5):
strDataDirectory = listOfDirectories[1]
strSecondDirectory = listOfDirectories[2]
strThirdDirectory = listOfDirectories[3]
strFourthDirectory = listOfDirectories[4]
strFifthDirectory = listOfDirectories[5]
year = strFifthDirectory[0:4]
strProposal = None
strBeamline = None
if (strDataDirectory == "data") and (strSecondDirectory == "gz"):
if strThirdDirectory == "visitor":
strProposal = strFourthDirectory
strBeamline = strFifthDirectory
elif strFourthDirectory == "inhouse":
strProposal = strFifthDirectory
strBeamline = strThirdDirectory
else:
raise RuntimeError(
"Illegal path for EDHandlerESRFPyarchv1_0.createPyarchFilePath: {0}"
.format(_strESRFPath))
listOfRemainingDirectories = listOfDirectories[6:]
elif (strDataDirectory == "data") and (strSecondDirectory
== "visitor"):
strProposal = listOfDirectories[3]
strBeamline = listOfDirectories[4]
listOfRemainingDirectories = listOfDirectories[5:]
elif ((strDataDirectory == "data")
and (strSecondDirectory in listBeamlines)):
strBeamline = strSecondDirectory
strProposal = listOfDirectories[4]
listOfRemainingDirectories = listOfDirectories[5:]
if (strProposal != None) and (strBeamline != None):
strPyarchDNAFilePath = os.path.join(os.sep, "data")
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath,
"pyarch")
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, year)
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath,
strBeamline)
strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath,
strProposal)
for strDirectory in listOfRemainingDirectories:
strPyarchDNAFilePath = os.path.join(
strPyarchDNAFilePath, strDirectory)
if (strPyarchDNAFilePath is None):
EDVerbose.WARNING(
"EDHandlerESRFPyarchv1_0.createPyArchFilePath: path not converted for pyarch: %s "
% _strESRFPath)
return strPyarchDNAFilePath
def generateCommands(self, _xsDataInputH5ToCBF):
"""
This method creates a list of commands for the converter
"""
self.DEBUG("EDPluginH5ToCBFv1_1.generateCommands")
hdf5File = _xsDataInputH5ToCBF.hdf5File.path.value
directory = os.path.dirname(hdf5File)
prefix = EDUtilsImage.getPrefix(hdf5File)
if _xsDataInputH5ToCBF.imageNumber is not None:
imageNumber = _xsDataInputH5ToCBF.imageNumber.value
if _xsDataInputH5ToCBF.hdf5ImageNumber is None:
hdf5ImageNumber = imageNumber
else:
hdf5ImageNumber = _xsDataInputH5ToCBF.hdf5ImageNumber.value
if "master" in hdf5File:
masterFile = hdf5File
else:
if EDUtilsPath.isEMBL():
masterFile = os.path.join(directory, prefix + "_master.h5".format(hdf5ImageNumber))
else:
masterFile = os.path.join(directory, prefix + "_{0}_master.h5".format(hdf5ImageNumber))
if _xsDataInputH5ToCBF.forcedOutputImageNumber is not None:
CBFFileName = prefix + "_%04d" % _xsDataInputH5ToCBF.forcedOutputImageNumber.value + ".cbf"
imageNumberInHdf5File = imageNumber
else:
CBFFileName = prefix + "_%04d" % imageNumber + ".cbf"
imageNumberInHdf5File = imageNumber - hdf5ImageNumber + 1
tmpCBFFileName = "tmp_" + CBFFileName
if _xsDataInputH5ToCBF.forcedOutputDirectory is None:
self.CBFFile = os.path.join(directory, CBFFileName)
else:
forcedOutputDirectory = self.dataInput.forcedOutputDirectory.path.value
if not os.path.exists(forcedOutputDirectory):
os.makedirs(forcedOutputDirectory, 0o755)
self.CBFFile = os.path.join(forcedOutputDirectory, CBFFileName)
scriptCommandLine = "{0} {1} {2}".format(masterFile, imageNumberInHdf5File, self.CBFFile)
elif _xsDataInputH5ToCBF.startImageNumber is not None and _xsDataInputH5ToCBF.endImageNumber is not None:
startImageNumber = _xsDataInputH5ToCBF.startImageNumber.value
endImageNumber = _xsDataInputH5ToCBF.endImageNumber.value
if _xsDataInputH5ToCBF.hdf5ImageNumber is None:
hdf5ImageNumber = startImageNumber
else:
hdf5ImageNumber = _xsDataInputH5ToCBF.hdf5ImageNumber.value
if "master" in hdf5File:
masterFile = hdf5File
else:
masterFile = os.path.join(directory, prefix + "_{0}_master.h5".format(hdf5ImageNumber))
CBFFileNamePrefix = prefix + "_"
if _xsDataInputH5ToCBF.forcedOutputDirectory is None:
CBFFilePath = os.path.join(directory, CBFFileNamePrefix)
else:
forcedOutputDirectory = self.dataInput.forcedOutputDirectory.path.value
if not os.path.exists(forcedOutputDirectory):
os.makedirs(forcedOutputDirectory, 0o755)
CBFFilePath = os.path.join(forcedOutputDirectory, CBFFileNamePrefix)
scriptCommandLine = "{0} {1}:{2} {3}".format(masterFile, startImageNumber, endImageNumber, CBFFilePath)
self.CBFFileTemplate = CBFFilePath + "######.cbf"
return scriptCommandLine
def preProcess(self, _edObject=None):
EDPluginExecProcessScript.preProcess(self)
self.DEBUG("EDPluginMinimalXDS.preProcess")
xds_input = os.path.abspath(self.dataInput.input_file.value)
workingDirectory = self.getWorkingDirectory()
if not os.path.exists(workingDirectory):
os.makedirs(workingDirectory, 0o755)
shutil.copy(xds_input, workingDirectory)
# our new xds file
xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
parsed_config = parse_xds_file(xds_file)
file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
# Find out the real path for the template
file_template_real_path = os.path.join(os.path.dirname(xds_input), file_template)
# get the real directory the files are in by getting the real
# path of the first image
first_image_no = parsed_config['DATA_RANGE='][0]
first_image = _template_to_image(file_template_real_path, first_image_no)
parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = file_template_real_path
# perhaps modify some params
job = self.dataInput.job
maxproc = self.dataInput.maxproc
maxjobs = self.dataInput.maxjobs
resolution_range = self.dataInput.resolution_range
friedels_law = self.dataInput.friedels_law
spot_range = self.dataInput.spot_range
spacegroup = self.dataInput.spacegroup
unit_cell = self.dataInput.unit_cell
self.DEBUG('requested spot range is {0}'.format(spot_range))
if job is not None:
parsed_config["JOB="] = job.value
if maxproc is not None:
parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
if maxjobs is not None:
parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
if resolution_range is not None and len(resolution_range) != 0:
parsed_config["INCLUDE_RESOLUTION_RANGE="] = [x.value for x in resolution_range]
if friedels_law is not None:
if friedels_law.value:
parsed_config["FRIEDEL'S_LAW="] = "TRUE"
else:
parsed_config["FRIEDEL'S_LAW="] = "FALSE"
if self.dataInput.start_image is not None:
start_image = self.dataInput.start_image.value
parsed_config['DATA_RANGE='][0] = start_image
else:
start_image = parsed_config['DATA_RANGE='][0]
if self.dataInput.end_image is not None:
end_image = self.dataInput.end_image.value
parsed_config['DATA_RANGE='][1] = end_image
else:
end_image = parsed_config['DATA_RANGE='][1]
spot_range_list = list()
if spot_range is not None and len(spot_range) > 0:
spot_range_list = []
for srange in spot_range:
spot_range_list.append([srange.begin, srange.end])
spot_range_list = self.checkSpotRanges(spot_range_list, start_image, end_image)
elif 'SPOT_RANGE=' in parsed_config:
spot_range_list = self.checkSpotRanges(parsed_config['SPOT_RANGE='], start_image, end_image)
else:
spot_range_list = [[start_image, end_image]]
self.DEBUG('setting the spot range to {0}'.format(spot_range_list))
parsed_config['SPOT_RANGE='] = spot_range_list
# Check background range
if 'BACKGROUND_RANGE=' in parsed_config:
background_range = parsed_config['BACKGROUND_RANGE=']
if background_range[0] < start_image:
background_range[0] = start_image
background_range[1] += start_image - 1
if background_range[1] > end_image:
background_range[1] = end_image
parsed_config['BACKGROUND_RANGE='] = background_range
# unit cell might be an empty string or some other crazy stuff
# we need 6 floats/ints
if unit_cell is not None:
ucells = unit_cell.value.split(' ')
if len(ucells) != 6:
unit_cell = None
else:
try:
if any(float(x) == 0 for x in ucells):
unit_cell = None
except ValueError:
unit_cell = None
# both need to be specified
if spacegroup is not None and unit_cell is not None:
self.DEBUG('specific spacegroup requested: {0}'.format(spacegroup.value))
self.DEBUG('specific unit cell requested: {0}'.format(unit_cell.value))
parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
# Check if this is ok
parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value
# For [XY]-GEO_CORR files, link them in the cwd and fix their paths
if 'X-GEO_CORR=' in parsed_config:
xgeo = parsed_config['X-GEO_CORR='][0]
xgeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(xgeo))
if not os.path.exists(xgeo_path):
os.symlink(xgeo, xgeo_path)
parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
if 'Y-GEO_CORR=' in parsed_config:
ygeo = parsed_config['Y-GEO_CORR='][0]
ygeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(ygeo))
if not os.path.exists(ygeo_path):
os.symlink(ygeo, ygeo_path)
parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)
# Neggia plugin
if not "LIB=" in parsed_config and self.pathToNeggiaPlugin is not None:
parsed_config["LIB="] = self.pathToNeggiaPlugin
# Max no processors
if not EDUtilsPath.isEMBL():
parsed_config['MAXIMUM_NUMBER_OF_PROCESSORS='] = str(self.maxNoProcessors)
parsed_config['MAXIMUM_NUMBER_OF_JOBS='] = 1
# Save back the changes
dump_xds_file(xds_file, parsed_config)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlDozorv1_0.process")
self.sendMessageToMXCuBE("Processing started...", "info")
EDUtilsParallel.initializeNbThread()
xsDataResultControlDozor = XSDataResultControlDozor()
# Check if connection to ISPyB needed
if self.dataInput.dataCollectionId is not None:
edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
if self.batchSize is None:
batchSize = ispybDataCollection.numberOfImages
else:
batchSize = self.batchSize
if batchSize > self.maxBatchSize:
batchSize = self.maxBatchSize
if abs(ispybDataCollection.overlap) > 1:
self.hasOverlap = True
self.overlap = ispybDataCollection.overlap
dictImage = self.createImageDictFromISPyB(ispybDataCollection)
else:
# No connection to ISPyB, take parameters from input
if self.dataInput.batchSize is None:
batchSize = self.maxBatchSize
else:
batchSize = self.dataInput.batchSize.value
dictImage = self.createImageDict(self.dataInput)
self.screen("Dozor batch size: {0}".format(batchSize))
if self.dataInput.hdf5BatchSize is not None:
self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
# Convert HDF5 images to CBF
self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
if self.doRadiationDamage:
self.cbfTempDir = None
else:
self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
for listBatch in listAllBatches:
# Read the header from the first image in the batch
xsDataFile = dictImage[listBatch[0]]
edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = xsDataFile
edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
edPluginControlReadImageHeader.executeSynchronous()
subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
xsDataInputDozor = XSDataInputDozor()
beam = subWedge.experimentalCondition.beam
detector = subWedge.experimentalCondition.detector
goniostat = subWedge.experimentalCondition.goniostat
xsDataInputDozor.detectorType = detector.type
xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
xsDataInputDozor.spotSize = XSDataInteger(3)
xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
# xsDataInputDozor.fractionPolatization : XSDataDouble optional
orgx = detector.beamPositionY.value / detector.pixelSizeY.value
orgy = detector.beamPositionX.value / detector.pixelSizeX.value
xsDataInputDozor.orgx = XSDataDouble(orgx)
xsDataInputDozor.orgy = XSDataDouble(orgy)
xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
# xsDataInputDozor.imageStep : XSDataDouble optional
xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
xsDataInputDozor.firstImageNumber = subWedge.image[0].number
xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
if self.hasOverlap:
xsDataInputDozor.overlap = XSDataAngle(self.overlap)
strFileName = subWedge.image[0].path.value
strPrefix = EDUtilsImage.getPrefix(strFileName)
strSuffix = EDUtilsImage.getSuffix(strFileName)
if EDUtilsPath.isEMBL():
strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
elif self.hasHdf5Prefix and not self.hasOverlap:
strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
else:
strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
edPluginDozor.dataInput = xsDataInputDozor
edPluginDozor.execute()
edPluginDozor.synchronize()
indexImage = 0
imageDozorBatchList = []
for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
xsDataControlImageDozor = XSDataControlImageDozor()
xsDataControlImageDozor.number = xsDataResultDozor.number
xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
xsDataControlImageDozor.angle = xsDataResultDozor.angle
xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
if xsDataResultControlDozor.inputDozor is None:
xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
indexImage += 1
dozorSpotListShape = []
dozorSpotList = []
spotFile = None
if xsDataControlImageDozor.spotFile is not None:
spotFile = xsDataControlImageDozor.spotFile.path.value
if os.path.exists(spotFile):
numpyArray = numpy.loadtxt(spotFile, skiprows=3)
dozorSpotList = base64.b64encode(numpyArray.tostring())
dozorSpotListShape.append(numpyArray.shape[0])
if len(numpyArray.shape) > 1:
dozorSpotListShape.append(numpyArray.shape[1])
imageDozorDict = {"index": xsDataControlImageDozor.number.value,
"imageName": xsDataControlImageDozor.image.path.value,
"dozor_score": xsDataControlImageDozor.mainScore.value,
"dozorSpotsNumOf" : xsDataControlImageDozor.spotsNumOf.value,
"dozorSpotFile": spotFile,
"dozorSpotList" : dozorSpotList,
"dozorSpotListShape": dozorSpotListShape,
"dozorSpotsIntAver": xsDataControlImageDozor.spotsIntAver.value,
"dozorSpotsResolution": xsDataControlImageDozor.spotsResolution.value
}
imageDozorBatchList.append(imageDozorDict)
xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
xsDataResultControlDozor.pngPlots = edPluginDozor.dataOutput.pngPlots
self.sendResultToMXCuBE(imageDozorBatchList)
self.sendMessageToMXCuBE("Batch processed")
self.dataOutput = xsDataResultControlDozor
if self.cbfTempDir is not None:
if self.dataInput.keepCbfTmpDirectory is not None and self.dataInput.keepCbfTmpDirectory.value:
self.dataOutput.pathToCbfDirectory = XSDataFile(XSDataString(self.cbfTempDir))
else:
shutil.rmtree(self.cbfTempDir)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
elif EDUtilsPath.isMAXIV():
template = ispybDataCollection.fileTemplate
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isMAXIV():
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
if self.doAnom:
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
if self.doNoanom:
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
if self.doNoanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
