def create_integration_id(self, datacollect_id, comments):
autoproc_status = edFactoryPlugin.loadPlugin(
'EDPluginISPyBStoreAutoProcStatusv1_4')
status_input = XSDataInputStoreAutoProcStatus()
status_input.dataCollectionId = datacollect_id
status_input.anomalous = True
# needed even if we only want to get an integration ID?
status_data = AutoProcStatus()
status_data.step = "Indexing"
status_data.status = "Launched"
status_data.comments = comments
# Program
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=None,
status="RUNNING",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
status_input.AutoProcProgram = autoProcProgram
status_input.AutoProcStatus = status_data
autoproc_status.dataInput = status_input
# get our EDNAproc status id
autoproc_status.executeSynchronous()
return (autoproc_status.dataOutput.autoProcIntegrationId,
autoproc_status.dataOutput.autoProcProgramId)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlCrystFELv1_0.postProcess")
"""
autoProcScalingStatisticsId : integer optional
scalingStatisticsType : string
comments : string
resolutionLimitLow : float
resolutionLimitHigh : float
rMerge : float
rMeasWithinIPlusIMinus : float
rMeasAllIPlusIMinus : float
rPimWithinIPlusIMinus : float
rPimAllIPlusIMinus : float
fractionalPartialBias : float
nTotalObservations : integer
ntotalUniqueObservations : integer
meanIOverSigI : float
completeness : float
multiplicity : float
anomalousCompleteness : float
anomalousMultiplicity : float
recordTimeStamp : string
anomalous : boolean
autoProcScalingId : integer
ccHalf : float
ccAno : float
sigAno : float
isa : float
completenessSpherical : float
anomalousCompletenessSpherical : float
completenessEllipsoidal : float
anomalousCompletenessEllipsoidal : float
"""
"""
overallCompl : XSDataDouble
overallRed : XSDataDouble
overallSnr : XSDataDouble
overallRsplit : XSDataDouble
overallCC
"""
output = AutoProcContainer()
autoproc = AutoProc()
autoproc.spaceGroup = str(self.dataInput.spaceGroup.value)
output.AutoProc = autoproc
scaling_container = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
scaling_container.AutoProcScaling = scaling
#Open the cell file and get the cell parameters
cell_file = open(self.dataInput.cellFile.value, "r")
cell_params = {}
for line in cell_file.readlines():
if line.startswith("a ="):
cell_params["a"] = float(line[4:9])
elif line.startswith("b ="):
cell_params["b"] = float(line[4:9])
elif line.startswith("c ="):
cell_params["c"] = float(line[4:9])
elif line.startswith("al ="):
cell_params["alpha"] = float(line[4:9])
elif line.startswith("be ="):
cell_params["beta"] = float(line[4:9])
elif line.startswith("ga ="):
cell_params["gamma"] = float(line[4:9])
cell_file.close()
autoproc.refinedCell_a = cell_params["a"]
autoproc.refinedCell_b = cell_params["b"]
autoproc.refinedCell_c = cell_params["c"]
autoproc.refinedCell_alpha = cell_params["alpha"]
autoproc.refinedCell_beta = cell_params["beta"]
autoproc.refinedCell_gamma = cell_params["gamma"]
overall_stats = AutoProcScalingStatistics()
overall_stats.scalingStatisticsType = "innerShell"
overall_stats.ccHalf = float(
self.post_process_plugin.dataOutput.overallCC.value)
overall_stats.resolutionLimitLow = float(
self.post_process_plugin.dataOutput.resolutionLimitLow.value)
overall_stats.resolutionLimitHigh = float(
self.post_process_plugin.dataOutput.resolutionLimitHigh.value)
overall_stats.completeness = float(
self.post_process_plugin.dataOutput.overallCompl.value)
#TODO get rMerge: otherwise results are not displayed in EXI
overall_stats.rMerge = 0
scaling_container.AutoProcScalingStatistics.append(overall_stats)
integration_container = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.dataCollectionId.value
integration_container.Image = image
integration = AutoProcIntegration()
if self.integration_id is not None:
integration.autoProcIntegrationId = self.integration_id
integration.cell_a = cell_params["a"]
integration.cell_b = cell_params["b"]
integration.cell_c = cell_params["c"]
integration.cell_alpha = cell_params["alpha"]
integration.cell_beta = cell_params["beta"]
integration.cell_gamma = cell_params["gamma"]
integration.anomalous = 0
# done with the integration
integration_container.AutoProcIntegration = integration
scaling_container.AutoProcIntegrationContainer = integration_container
program_container = AutoProcProgramContainer()
program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=self.program_id,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
output.AutoProcProgramContainer = program_container
output.AutoProcScalingContainer = scaling_container
for log_file in self.post_process_plugin.dataOutput.logFiles:
pathToLogFile = log_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToLogFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Log"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
for data_file in self.post_process_plugin.dataOutput.dataFiles:
pathToDataFile = data_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToDataFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Result"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
self.store_autoproc_plugin.dataInput = ispyb_input
self.store_autoproc_plugin.executeSynchronous()
if self.store_autoproc_plugin.isFailure():
self.ERROR("Could not upload results to ispyb!")
else:
self.hasUploadedAnomResultsToISPyB = True
self.screen("Results uploaded to ISPyB")
xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment()
xsDataInput.newComment = XSDataString(
self.post_process_plugin.dataOutput.comment.value)
xsDataInput.dataCollectionId = self.dataInput.dataCollectionId
self.store_dc_comment_plugin.dataInput = xsDataInput
self.executePluginSynchronous(self.store_dc_comment_plugin)
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId,
integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(
edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToSummaryFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToHtmlFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToLogFile)
# Copy all log files to results:
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(
pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToDataFile)
shutil.copy(
pathToDataFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToDataFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin(
"EDPluginISPyBStoreAutoProcv1_4",
"EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(
trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value,
os.path.join(self.resultsDirectory, newFileName))
return successUpload
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
# Copy all log files to results:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
return successUpload
def testCreateAutoProcProgram(self):
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram()
print(autoProcProgram.marshal())
def testCreateAutoProcProgram(self):
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram()
print(autoProcProgram.marshal())
