def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
self.edPluginExecXia2DIALSAnom.synchronize()
self.edPluginExecXia2DIALSNoanom.synchronize()
strMessage = ""
if self.getListOfWarningMessages() != []:
strMessage += "Warning messages: \n\n"
for strWarningMessage in self.getListOfWarningMessages():
strMessage += strWarningMessage + "\n\n"
if self.getListOfErrorMessages() != []:
strMessage += "Error messages: \n\n"
for strErrorMessage in self.getListOfErrorMessages():
strMessage += strErrorMessage + "\n\n"
if self.isFailure():
self.screen("XIA2_DIALS processing ended with errors!")
if strMessage != "":
self.screen(
"Warning and/or error messages: \n{0}.".format(strMessage))
self.timeEnd = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Upload program failure status to ISPyB
# anom
self.screen("Setting anom program status to failed in ISPyB.")
if not self.hasUploadedAnomResultsToISPyB:
self.screen(
"Setting anom program status to failed in ISPyB.")
EDHandlerXSDataISPyBv1_4.setIspybToFailed(
self,
dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if self.doAnomAndNonanom:
# noanom
if not self.hasUploadedNoanomResultsToISPyB:
self.screen(
"Setting noanom program status to failed in ISPyB."
)
EDHandlerXSDataISPyBv1_4.setIspybToFailed(
self,
dataCollectionId=self.dataInput.dataCollectionId.
value,
autoProcIntegrationId=self.
autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
strMessage = ""
if self.getListOfWarningMessages() != []:
strMessage += "Warning messages: \n\n"
for strWarningMessage in self.getListOfWarningMessages():
strMessage += strWarningMessage + "\n\n"
if self.getListOfErrorMessages() != []:
strMessage += "Error messages: \n\n"
for strErrorMessage in self.getListOfErrorMessages():
strMessage += strErrorMessage + "\n\n"
if self.isFailure():
self.timeEnd = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Upload program status to ISPyB
# anom
if self.doAnom:
if not self.hasUploadedAnomResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if not self.hasUploadedAnomStaranisoResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if self.doNoanom:
# noanom
if not self.hasUploadedNoanomResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if not self.hasUploadedNoanomStaranisoResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
def preProcess(self, _edObject=None):
"""
Loads the ISPyB execution plugin and prepares the input data
"""
EDPluginControl.preProcess(self, _edObject)
self.DEBUG("EDPluginControlISPyBv1_4.preProcess...")
if (self.hasDataInput("shortComments")):
self.strShortComments = self.getDataInput("shortComments")[0].getValue()
if (self.hasDataInput("comments")):
self.strComments = self.getDataInput("comments")[0].getValue()
if (self.hasDataInput("statusMessage")):
self.strStatusMessage = self.getDataInput("statusMessage")[0].getValue()
self.edPluginExecISPyB = self.loadPlugin(self.edStringPluginExecISPyBName)
try:
xsDataInputISPyBStoreScreening = EDHandlerXSDataISPyBv1_4.generateXSDataInputISPyBStoreScreening(self.getDataInput(),
self.strStatusMessage, \
self.strShortComments, \
self.strComments)
except Exception as error:
raise
# This exception handling needs to be rethought, see bug #43.
errorMessage = "EDPluginControlISPyBv1_4.preProcess: Unexpected error in ISPyB handler: %r" % error
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
self.edPluginExecISPyB.setDataInput(xsDataInputISPyBStoreScreening)
def preProcess(self, _edObject=None):
"""
Loads the ISPyB execution plugin and prepares the input data
"""
EDPluginControl.preProcess(self, _edObject)
EDVerbose.DEBUG("EDPluginControlISPyBv1_4.preProcess...")
if (self.hasDataInput("shortComments")):
self.strShortComments = self.getDataInput(
"shortComments")[0].getValue()
if (self.hasDataInput("comments")):
self.strComments = self.getDataInput("comments")[0].getValue()
if (self.hasDataInput("statusMessage")):
self.strStatusMessage = self.getDataInput(
"statusMessage")[0].getValue()
self.edPluginExecISPyB = self.loadPlugin(
self.edStringPluginExecISPyBName)
try:
xsDataInputISPyBStoreScreening = EDHandlerXSDataISPyBv1_4.generateXSDataInputISPyBStoreScreening(self.getDataInput(),
self.strStatusMessage, \
self.strShortComments, \
self.strComments)
except Exception, error:
raise
# This exception handling needs to be rethought, see bug #43.
errorMessage = "EDPluginControlISPyBv1_4.preProcess: Unexpected error in ISPyB handler: %r" % error
EDVerbose.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError, errorMessage
def create_integration_id(self, datacollect_id, comments):
autoproc_status = edFactoryPlugin.loadPlugin(
'EDPluginISPyBStoreAutoProcStatusv1_4')
status_input = XSDataInputStoreAutoProcStatus()
status_input.dataCollectionId = datacollect_id
status_input.anomalous = True
# needed even if we only want to get an integration ID?
status_data = AutoProcStatus()
status_data.step = "Indexing"
status_data.status = "Launched"
status_data.comments = comments
# Program
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=None,
status="RUNNING",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
status_input.AutoProcProgram = autoProcProgram
status_input.AutoProcStatus = status_data
autoproc_status.dataInput = status_input
# get our EDNAproc status id
autoproc_status.executeSynchronous()
return (autoproc_status.dataOutput.autoProcIntegrationId,
autoproc_status.dataOutput.autoProcProgramId)
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
self.edPluginExecAutoPROCAnom.synchronize()
self.edPluginExecAutoPROCNoanom.synchronize()
strMessage = ""
if self.getListOfWarningMessages() != []:
strMessage += "Warning messages: \n\n"
for strWarningMessage in self.getListOfWarningMessages():
strMessage += strWarningMessage + "\n\n"
if self.getListOfErrorMessages() != []:
strMessage += "Error messages: \n\n"
for strErrorMessage in self.getListOfErrorMessages():
strMessage += strErrorMessage + "\n\n"
if self.isFailure():
self.timeEnd = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Upload program status to ISPyB
# anom
if not self.hasUploadedAnomResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if not self.hasUploadedAnomStaranisoResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if self.doAnomAndNonanom:
# noanom
if not self.hasUploadedNoanomResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if not self.hasUploadedNoanomStaranisoResultsToISPyB:
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
def testGetBestWilsonPlotPath(self):
strReferenceInputControlISPyBFile = EDUtilsPath.mergePath(self.strDataPath, "XSDataInputControlISPyB_reference.xml")
strPath = os.path.join(self.strDataPath, strReferenceInputControlISPyBFile)
strXMLIndexingInput = EDUtilsFile.readFileAndParseVariables(strPath, self.dictReplace)
xsDataInputControlISPyB = XSDataInputControlISPyB.parseString(strXMLIndexingInput)
xsDataCharacterisationResult = xsDataInputControlISPyB.characterisationResult
strPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataCharacterisationResult)
EDAssert.equal(True, strPath.endswith("B.jpg"), "Wilson plot path extracted from characterisation results")
def finallyProcess(self, _edObject=None):
EDPluginControl.finallyProcess(self)
self.edPluginExecXia2DIALSAnom.synchronize()
self.edPluginExecXia2DIALSNoanom.synchronize()
strMessage = ""
if self.getListOfWarningMessages() != []:
strMessage += "Warning messages: \n\n"
for strWarningMessage in self.getListOfWarningMessages():
strMessage += strWarningMessage + "\n\n"
if self.getListOfErrorMessages() != []:
strMessage += "Error messages: \n\n"
for strErrorMessage in self.getListOfErrorMessages():
strMessage += strErrorMessage + "\n\n"
if self.isFailure():
self.screen("XIA2_DIALS processing ended with errors!")
if strMessage != "":
self.screen("Warning and/or error messages: \n{0}.".format(strMessage))
self.timeEnd = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Upload program failure status to ISPyB
# anom
self.screen("Setting anom program status to failed in ISPyB.")
if not self.hasUploadedAnomResultsToISPyB:
self.screen("Setting anom program status to failed in ISPyB.")
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdAnom,
autoProcProgramId=self.autoProcProgramIdAnom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
if self.doAnomAndNonanom:
# noanom
if not self.hasUploadedNoanomResultsToISPyB:
self.screen("Setting noanom program status to failed in ISPyB.")
EDHandlerXSDataISPyBv1_4.setIspybToFailed(self, dataCollectionId=self.dataInput.dataCollectionId.value,
autoProcIntegrationId=self.autoProcIntegrationIdNoanom,
autoProcProgramId=self.autoProcProgramIdNoanom,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart,
timeEnd=self.timeEnd)
def testGenerateXSDataInputISPyB(self):
"""
This method is testing the generation of the XSDataInputISPyB object given a XSDataIndexingInput object.
"""
strReferenceInputControlISPyBFile = EDUtilsPath.mergePath(self.strDataPath, "XSDataInputControlISPyB_reference.xml")
strPath = os.path.join(self.strDataPath, strReferenceInputControlISPyBFile)
strXMLIndexingInput = EDUtilsFile.readFileAndParseVariables(strPath, self.dictReplace)
xsDataInputControlISPyB = XSDataInputControlISPyB.parseString(strXMLIndexingInput)
xsDataInputISPyB = EDHandlerXSDataISPyBv1_4.generateXSDataInputISPyBStoreScreening(xsDataInputControlISPyB)
strReferenceInputISPyBFile = EDUtilsPath.mergePath(self.strDataPath, "XSDataInputISPyB_reference.xml")
strReferencePath = os.path.join(self.strDataPath, strReferenceInputISPyBFile)
strXMLInputISPyBReference = EDUtilsFile.readFileAndParseVariables(strReferencePath, self.dictReplace)
EDFactoryPluginStatic.loadModule("XSDataISPyBv1_4")
def testGetBestWilsonPlotPath(self):
strReferenceInputControlISPyBFile = EDUtilsPath.mergePath(
self.strDataPath, "XSDataInputControlISPyB_reference.xml")
strPath = os.path.join(self.strDataPath,
strReferenceInputControlISPyBFile)
strXMLIndexingInput = EDUtilsFile.readFileAndParseVariables(
strPath, self.dictReplace)
xsDataInputControlISPyB = XSDataInputControlISPyB.parseString(
strXMLIndexingInput)
xsDataCharacterisationResult = xsDataInputControlISPyB.characterisationResult
strPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(
xsDataCharacterisationResult)
EDAssert.equal(
True, strPath.endswith("B.jpg"),
"Wilson plot path extracted from characterisation results")
def testGenerateXSDataInputISPyB(self):
"""
This method is testing the generation of the XSDataInputISPyB object given a XSDataIndexingInput object.
"""
strReferenceInputControlISPyBFile = EDUtilsPath.mergePath(
self.strDataPath, "XSDataInputControlISPyB_reference.xml")
strPath = os.path.join(self.strDataPath,
strReferenceInputControlISPyBFile)
strXMLIndexingInput = EDUtilsFile.readFileAndParseVariables(
strPath, self.dictReplace)
xsDataInputControlISPyB = XSDataInputControlISPyB.parseString(
strXMLIndexingInput)
xsDataInputISPyB = EDHandlerXSDataISPyBv1_4.generateXSDataInputISPyBStoreScreening(
xsDataInputControlISPyB)
strReferenceInputISPyBFile = EDUtilsPath.mergePath(
self.strDataPath, "XSDataInputISPyB_reference.xml")
strReferencePath = os.path.join(self.strDataPath,
strReferenceInputISPyBFile)
strXMLInputISPyBReference = EDUtilsFile.readFileAndParseVariables(
strReferencePath, self.dictReplace)
EDFactoryPluginStatic.loadModule("XSDataISPyBv1_4")
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
if self.dataInput.startImageNumber is not None:
imageNoStart = self.dataInput.startImageNumber.value
if self.dataInput.endImageNumber is not None:
imageNoEnd = self.dataInput.endImageNumber.value
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if imageNoStart is None:
imageNoStart = ispybDataCollection.startImageNumber
if imageNoEnd is None:
imageNoEnd = ispybDataCollection.startImageNumber + \
ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"XDSAPP", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
self.timeStart = time.localtime()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
if self.doNoanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.doAnom:
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId,
integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(
edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToSummaryFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToHtmlFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToLogFile)
# Copy all log files to results:
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(
pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToDataFile)
shutil.copy(
pathToDataFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToDataFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin(
"EDPluginISPyBStoreAutoProcv1_4",
"EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(
trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value,
os.path.join(self.resultsDirectory, newFileName))
return successUpload
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doNoanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
if self.doAnom:
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
# Prepare input to execution plugin
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
if self.doAnomAndNonanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
# if isH5:
# masterFilePath = os.path.join(directory,
# self.eiger_template_to_master(template))
# xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
# if self.doAnomAndNonanom:
# xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
# else:
# xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
# if self.doAnomAndNonanom:
# xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doAnomAndNonanom:
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
if self.doAnomAndNonanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doAnomAndNonanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
if self.bIsEigerDetector:
xsDataResultCharacterisation = self.makeNumberOfImagesMultipleOf100(xsDataResultCharacterisation)
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if self.createDNAFileDirectory(strPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPathToDNAFileDirectory
)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory
)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan is not None and xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(
XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory"
)
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
strBestWilsonPlotPyarchPath = None
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(
strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)
)
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = (
self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
)
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin(
"EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath"
)
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
# Only for the ESRF:
if EDUtilsPath.isESRF():
# For EXI: create workflow entry with one workflow step
self.edPluginStoreWorkflow = self.loadPlugin(self.strPluginStoreWorkflow)
self.edPluginStoreWorkflowStep = self.loadPlugin(self.strPluginStoreWorkflowStep)
self.edPluginUpdateDataCollectionGroupWorkflowId = self.loadPlugin(
self.strUpdateDataCollectionGroupWorkflowId
)
xsDataISPyBWorkflow = XSDataISPyBWorkflow()
xsDataISPyBWorkflow.workflowType = XSDataString("Characterisation")
xsDataISPyBWorkflow.workflowTitle = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflow = XSDataInputISPyBStoreWorkflow()
xsDataInputISPyBStoreWorkflow.workflow = xsDataISPyBWorkflow
self.edPluginStoreWorkflow.dataInput = xsDataInputISPyBStoreWorkflow
self.edPluginStoreWorkflow.executeSynchronous()
if (
self.edPluginStoreWorkflow.dataOutput is not None
and self.edPluginStoreWorkflow.dataOutput.workflowId is not None
):
workflowId = self.edPluginStoreWorkflow.dataOutput.workflowId.value
# Update data collection group
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId = (
XSDataInputISPyBUpdateDataCollectionGroupWorkflowId()
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileLocation = XSDataString(
os.path.dirname(self.xsDataFirstImage.path.value)
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileName = XSDataString(
os.path.basename(self.xsDataFirstImage.path.value)
)
self.edPluginUpdateDataCollectionGroupWorkflowId.dataInput = (
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId
)
self.edPluginUpdateDataCollectionGroupWorkflowId.executeSynchronous()
xsDataInputISPyBStoreWorkflowStep = XSDataInputISPyBStoreWorkflowStep()
xsDataInputISPyBStoreWorkflowStep.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBStoreWorkflowStep.workflowStepType = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflowStep.status = XSDataString("Success")
if strBestWilsonPlotPyarchPath is not None:
xsDataInputISPyBStoreWorkflowStep.imageResultFilePath = XSDataString(
strBestWilsonPlotPyarchPath
)
xsDataInputISPyBStoreWorkflowStep.htmlResultFilePath = XSDataString(strPyArchPathToDNAFileDirectory)
if self.edPluginExecSimpleHTML.dataOutput is not None:
strResultFilePath = self.edPluginExecSimpleHTML.dataOutput.pathToJsonFile.path.value
# strPyarchResultFilePath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strResultFilePath)
xsDataInputISPyBStoreWorkflowStep.resultFilePath = XSDataString(strResultFilePath)
self.edPluginStoreWorkflowStep.dataInput = xsDataInputISPyBStoreWorkflowStep
self.edPluginStoreWorkflowStep.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
elif EDUtilsPath.isMAXIV():
template = ispybDataCollection.fileTemplate
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isMAXIV():
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
if self.doAnom:
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
if self.doNoanom:
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
if self.doNoanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlCrystFELv1_0.postProcess")
"""
autoProcScalingStatisticsId : integer optional
scalingStatisticsType : string
comments : string
resolutionLimitLow : float
resolutionLimitHigh : float
rMerge : float
rMeasWithinIPlusIMinus : float
rMeasAllIPlusIMinus : float
rPimWithinIPlusIMinus : float
rPimAllIPlusIMinus : float
fractionalPartialBias : float
nTotalObservations : integer
ntotalUniqueObservations : integer
meanIOverSigI : float
completeness : float
multiplicity : float
anomalousCompleteness : float
anomalousMultiplicity : float
recordTimeStamp : string
anomalous : boolean
autoProcScalingId : integer
ccHalf : float
ccAno : float
sigAno : float
isa : float
completenessSpherical : float
anomalousCompletenessSpherical : float
completenessEllipsoidal : float
anomalousCompletenessEllipsoidal : float
"""
"""
overallCompl : XSDataDouble
overallRed : XSDataDouble
overallSnr : XSDataDouble
overallRsplit : XSDataDouble
overallCC
"""
output = AutoProcContainer()
autoproc = AutoProc()
autoproc.spaceGroup = str(self.dataInput.spaceGroup.value)
output.AutoProc = autoproc
scaling_container = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
scaling_container.AutoProcScaling = scaling
#Open the cell file and get the cell parameters
cell_file = open(self.dataInput.cellFile.value, "r")
cell_params = {}
for line in cell_file.readlines():
if line.startswith("a ="):
cell_params["a"] = float(line[4:9])
elif line.startswith("b ="):
cell_params["b"] = float(line[4:9])
elif line.startswith("c ="):
cell_params["c"] = float(line[4:9])
elif line.startswith("al ="):
cell_params["alpha"] = float(line[4:9])
elif line.startswith("be ="):
cell_params["beta"] = float(line[4:9])
elif line.startswith("ga ="):
cell_params["gamma"] = float(line[4:9])
cell_file.close()
autoproc.refinedCell_a = cell_params["a"]
autoproc.refinedCell_b = cell_params["b"]
autoproc.refinedCell_c = cell_params["c"]
autoproc.refinedCell_alpha = cell_params["alpha"]
autoproc.refinedCell_beta = cell_params["beta"]
autoproc.refinedCell_gamma = cell_params["gamma"]
overall_stats = AutoProcScalingStatistics()
overall_stats.scalingStatisticsType = "innerShell"
overall_stats.ccHalf = float(
self.post_process_plugin.dataOutput.overallCC.value)
overall_stats.resolutionLimitLow = float(
self.post_process_plugin.dataOutput.resolutionLimitLow.value)
overall_stats.resolutionLimitHigh = float(
self.post_process_plugin.dataOutput.resolutionLimitHigh.value)
overall_stats.completeness = float(
self.post_process_plugin.dataOutput.overallCompl.value)
#TODO get rMerge: otherwise results are not displayed in EXI
overall_stats.rMerge = 0
scaling_container.AutoProcScalingStatistics.append(overall_stats)
integration_container = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.dataCollectionId.value
integration_container.Image = image
integration = AutoProcIntegration()
if self.integration_id is not None:
integration.autoProcIntegrationId = self.integration_id
integration.cell_a = cell_params["a"]
integration.cell_b = cell_params["b"]
integration.cell_c = cell_params["c"]
integration.cell_alpha = cell_params["alpha"]
integration.cell_beta = cell_params["beta"]
integration.cell_gamma = cell_params["gamma"]
integration.anomalous = 0
# done with the integration
integration_container.AutoProcIntegration = integration
scaling_container.AutoProcIntegrationContainer = integration_container
program_container = AutoProcProgramContainer()
program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=self.program_id,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
output.AutoProcProgramContainer = program_container
output.AutoProcScalingContainer = scaling_container
for log_file in self.post_process_plugin.dataOutput.logFiles:
pathToLogFile = log_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToLogFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Log"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
for data_file in self.post_process_plugin.dataOutput.dataFiles:
pathToDataFile = data_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToDataFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Result"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
self.store_autoproc_plugin.dataInput = ispyb_input
self.store_autoproc_plugin.executeSynchronous()
if self.store_autoproc_plugin.isFailure():
self.ERROR("Could not upload results to ispyb!")
else:
self.hasUploadedAnomResultsToISPyB = True
self.screen("Results uploaded to ISPyB")
xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment()
xsDataInput.newComment = XSDataString(
self.post_process_plugin.dataOutput.comment.value)
xsDataInput.dataCollectionId = self.dataInput.dataCollectionId
self.store_dc_comment_plugin.dataInput = xsDataInput
self.executePluginSynchronous(self.store_dc_comment_plugin)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
else:
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doAnomAndNonanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doAnomAndNonanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def testCreateAutoProcProgram(self):
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram()
print(autoProcProgram.marshal())
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput(
).getResultCharacterisation()
self.xsDataResultMXCuBE.setCharacterisationResult(
xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput(
).getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(
xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(
self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(
strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(
XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(
xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if not EDUtilsPath.isALBA:
if (self.createDNAFileDirectory(strPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation,
strPathToDNAFileDirectory)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
strPathToDNAFileDirectory)
if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation,
strPyArchPathToDNAFileDirectory)
self.xsDataResultMXCuBE.setOutputFileDictionary(
xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage(
).getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary(
).getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption(
).getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML
is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(
XSDataFile(XSDataString(strPathToDNAFileDirectory)),
"dnaFileDirectory")
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult(
)
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult(
)
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar(
) is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar(
).getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(
xsDataResultCharacterisation)
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(
strPyArchPathToDNAFileDirectory,
os.path.basename(strBestWilsonPlotPath))
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(
os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(
os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath,
strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " %
strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath(
)
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(
strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin(
"EDPluginISPyBSetBestWilsonPlotPathv1_4",
"ISPyBSetBestWilsonPlotPath")
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if (self.createDNAFileDirectory(strPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPathToDNAFileDirectory)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory")
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath))
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin("EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath")
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
# Copy all log files to results:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
return successUpload
def testCreateAutoProcProgram(self):
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram()
print(autoProcProgram.marshal())