def createInputStoreAutoProc(dataCollectionId,
integrationId,
isAnomalous=False,
programId=None,
status="SUCCESS",
timeStart=None,
timeEnd=None,
processingCommandLine=None,
processingPrograms=None):
autoProcContainer = AutoProcContainer()
integrationContainer = AutoProcIntegrationContainer()
scalingContainer = AutoProcScalingContainer()
integration = AutoProcIntegration()
if integrationId is not None:
integration.autoProcIntegrationId = integrationId
integration.anomalous = isAnomalous
integrationContainer.AutoProcIntegration = integration
image = Image()
image.dataCollectionId = dataCollectionId
integrationContainer.Image = image
scalingContainer.AutoProcIntegrationContainer = integrationContainer
programContainer = AutoProcProgramContainer()
programContainer.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status=status,
timeStart=timeStart,
timeEnd=timeEnd,
processingCommandLine=processingCommandLine,
processingPrograms=processingPrograms)
autoProcContainer.AutoProcProgramContainer = programContainer
autoProcContainer.AutoProcScalingContainer = scalingContainer
inputStoreAutoProc = XSDataInputStoreAutoProc()
inputStoreAutoProc.AutoProcContainer = autoProcContainer
return inputStoreAutoProc
def createInputStoreAutoProc(dataCollectionId, integrationId, isAnomalous=False,
programId=None, status="SUCCESS", timeStart=None, timeEnd=None,
processingCommandLine=None, processingPrograms=None):
autoProcContainer = AutoProcContainer()
integrationContainer = AutoProcIntegrationContainer()
scalingContainer = AutoProcScalingContainer()
integration = AutoProcIntegration()
if integrationId is not None:
integration.autoProcIntegrationId = integrationId
integration.anomalous = isAnomalous
integrationContainer.AutoProcIntegration = integration
image = Image()
image.dataCollectionId = dataCollectionId
integrationContainer.Image = image
scalingContainer.AutoProcIntegrationContainer = integrationContainer
programContainer = AutoProcProgramContainer()
programContainer.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status=status, timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=processingCommandLine, processingPrograms=processingPrograms)
autoProcContainer.AutoProcProgramContainer = programContainer
autoProcContainer.AutoProcScalingContainer = scalingContainer
inputStoreAutoProc = XSDataInputStoreAutoProc()
inputStoreAutoProc.AutoProcContainer = autoProcContainer
return inputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = trunc + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
# Read the generated ISPyB xml file - if any
if edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
autoProcProgram.processingPrograms = "XIA2_DIALS"
autoProcProgram.processingStatus = True
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.pdf"
# Convert the xia2.html to xia2.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(pathToHtmlFile)))
xsDataInputHTML2PDF.paperSize = XSDataString("A4")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
if pathToLogFile.endswith(".log"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlCrystFELv1_0.postProcess")
"""
autoProcScalingStatisticsId : integer optional
scalingStatisticsType : string
comments : string
resolutionLimitLow : float
resolutionLimitHigh : float
rMerge : float
rMeasWithinIPlusIMinus : float
rMeasAllIPlusIMinus : float
rPimWithinIPlusIMinus : float
rPimAllIPlusIMinus : float
fractionalPartialBias : float
nTotalObservations : integer
ntotalUniqueObservations : integer
meanIOverSigI : float
completeness : float
multiplicity : float
anomalousCompleteness : float
anomalousMultiplicity : float
recordTimeStamp : string
anomalous : boolean
autoProcScalingId : integer
ccHalf : float
ccAno : float
sigAno : float
isa : float
completenessSpherical : float
anomalousCompletenessSpherical : float
completenessEllipsoidal : float
anomalousCompletenessEllipsoidal : float
"""
"""
overallCompl : XSDataDouble
overallRed : XSDataDouble
overallSnr : XSDataDouble
overallRsplit : XSDataDouble
overallCC
"""
output = AutoProcContainer()
autoproc = AutoProc()
autoproc.spaceGroup = str(self.dataInput.spaceGroup.value)
output.AutoProc = autoproc
scaling_container = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
scaling_container.AutoProcScaling = scaling
#Open the cell file and get the cell parameters
cell_file = open(self.dataInput.cellFile.value, "r")
cell_params = {}
for line in cell_file.readlines():
if line.startswith("a ="):
cell_params["a"] = float(line[4:9])
elif line.startswith("b ="):
cell_params["b"] = float(line[4:9])
elif line.startswith("c ="):
cell_params["c"] = float(line[4:9])
elif line.startswith("al ="):
cell_params["alpha"] = float(line[4:9])
elif line.startswith("be ="):
cell_params["beta"] = float(line[4:9])
elif line.startswith("ga ="):
cell_params["gamma"] = float(line[4:9])
cell_file.close()
autoproc.refinedCell_a = cell_params["a"]
autoproc.refinedCell_b = cell_params["b"]
autoproc.refinedCell_c = cell_params["c"]
autoproc.refinedCell_alpha = cell_params["alpha"]
autoproc.refinedCell_beta = cell_params["beta"]
autoproc.refinedCell_gamma = cell_params["gamma"]
overall_stats = AutoProcScalingStatistics()
overall_stats.scalingStatisticsType = "innerShell"
overall_stats.ccHalf = float(
self.post_process_plugin.dataOutput.overallCC.value)
overall_stats.resolutionLimitLow = float(
self.post_process_plugin.dataOutput.resolutionLimitLow.value)
overall_stats.resolutionLimitHigh = float(
self.post_process_plugin.dataOutput.resolutionLimitHigh.value)
overall_stats.completeness = float(
self.post_process_plugin.dataOutput.overallCompl.value)
#TODO get rMerge: otherwise results are not displayed in EXI
overall_stats.rMerge = 0
scaling_container.AutoProcScalingStatistics.append(overall_stats)
integration_container = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.dataCollectionId.value
integration_container.Image = image
integration = AutoProcIntegration()
if self.integration_id is not None:
integration.autoProcIntegrationId = self.integration_id
integration.cell_a = cell_params["a"]
integration.cell_b = cell_params["b"]
integration.cell_c = cell_params["c"]
integration.cell_alpha = cell_params["alpha"]
integration.cell_beta = cell_params["beta"]
integration.cell_gamma = cell_params["gamma"]
integration.anomalous = 0
# done with the integration
integration_container.AutoProcIntegration = integration
scaling_container.AutoProcIntegrationContainer = integration_container
program_container = AutoProcProgramContainer()
program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=self.program_id,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
output.AutoProcProgramContainer = program_container
output.AutoProcScalingContainer = scaling_container
for log_file in self.post_process_plugin.dataOutput.logFiles:
pathToLogFile = log_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToLogFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Log"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
for data_file in self.post_process_plugin.dataOutput.dataFiles:
pathToDataFile = data_file.path.value
pyarchFileName = os.path.join(self.pyarchDirectory,
os.path.basename(pathToDataFile))
#shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoproc_program_attachment = AutoProcProgramAttachment()
autoproc_program_attachment.fileName = os.path.basename(
pyarchFileName)
autoproc_program_attachment.filePath = os.path.dirname(
pyarchFileName)
autoproc_program_attachment.fileType = "Result"
program_container.addAutoProcProgramAttachment(
autoproc_program_attachment)
self.screen("Result file %s uploaded to ISPyB" %
os.path.basename(pyarchFileName))
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
self.store_autoproc_plugin.dataInput = ispyb_input
self.store_autoproc_plugin.executeSynchronous()
if self.store_autoproc_plugin.isFailure():
self.ERROR("Could not upload results to ispyb!")
else:
self.hasUploadedAnomResultsToISPyB = True
self.screen("Results uploaded to ISPyB")
xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment()
xsDataInput.newComment = XSDataString(
self.post_process_plugin.dataOutput.comment.value)
xsDataInput.dataCollectionId = self.dataInput.dataCollectionId
self.store_dc_comment_plugin.dataInput = xsDataInput
self.executePluginSynchronous(self.store_dc_comment_plugin)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId,
integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(
edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToSummaryFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToHtmlFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToLogFile)
# Copy all log files to results:
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(
pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToDataFile)
shutil.copy(
pathToDataFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToDataFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin(
"EDPluginISPyBStoreAutoProcv1_4",
"EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(
trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value,
os.path.join(self.resultsDirectory, newFileName))
return successUpload
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
# Copy all log files to results:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
return successUpload
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if edPluginExecAutoPROC.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecAutoPROC.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
autoProcProgram.processingPrograms = "autoPROC"
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachment.fileName = pyarchPdfFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "xscale_XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl):
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
f_in = open(pathToXdsAsciiHkl)
f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
f_out.writelines(f_in)
f_out.close()
f_in.close()
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
if isAnom:
self.edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
self.edPluginStoreAutoprocAnom.executeSynchronous()
else:
self.edPluginStoreAutoprocNoanom.dataInput = xsDataInputStoreAutoProc
self.edPluginStoreAutoprocNoanom.executeSynchronous()
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
if isStaraniso:
staranisoString = "_staraniso"
else:
staranisoString = ""
# Read the generated ISPyB xml file
pathToISPyBXML = None
if isStaraniso:
if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
if pathToISPyBXML is not None:
autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100
autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100
autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100
autoProcScalingStatistics.rPimAllIPlusIMinus *= 100
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
if self.dataInput.dataCollectionId is not None:
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
if isAnom:
autoProcIntegration.anomalous = True
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
else:
autoProcIntegration.anomalous = False
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
if self.reprocess:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess"
else:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgram.processingStatus = "SUCCESS"
# EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
# Check if summary_inlined.html exists
summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
if os.path.exists(summaryInlinedHtmlPath):
summaryName = "summary_inlined"
summaryHtmlPath = summaryInlinedHtmlPath
else:
summaryName = "summary"
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Upload summary.html
pathtoFile = summaryHtmlPath
pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
if not pyarchFile in self.listPyarchFile:
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
self.listPyarchFile.append(pyarchFile)
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
if summaryName == "summary":
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
if self.resultsDirectory:
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
if self.pyarchDirectory is not None:
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
autoProcProgramAttachmentPdf.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
if self.pyarchDirectory is not None:
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
with open(pathToXdsAsciiHkl, 'rb') as f_in:
with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
if self.resultsDirectory:
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add report.pdf
pathToRepordPdf = None
if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
if pathToRepordPdf is not None:
pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
if self.resultsDirectory:
shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchReportFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoprocAnom.executeSynchronous()
isSuccess = not edPluginStoreAutoprocAnom.isFailure()
if isSuccess:
self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
if isAnom:
if isStaraniso:
self.hasUploadedAnomStaranisoResultsToISPyB = True
else:
self.hasUploadedAnomResultsToISPyB = True
else:
if isStaraniso:
self.hasUploadedNoanomStaranisoResultsToISPyB = True
else:
self.hasUploadedNoanomResultsToISPyB = True
else:
self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
def postProcess(self, _edObject = None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlAutoproc.postProcess")
#Now that we have executed the whole thing we need to create
#the suitable ISPyB plugin input and serialize it to the file
#we've been given as input
output = AutoProcContainer()
# AutoProc attr
autoproc = AutoProc()
xdsout = self.xds_first.dataOutput
if xdsout.sg_number is not None: # and it should not
autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value]
autoproc.refinedCell_a = xdsout.cell_a.value
autoproc.refinedCell_b = xdsout.cell_b.value
autoproc.refinedCell_c = xdsout.cell_c.value
autoproc.refinedCell_alpha = xdsout.cell_alpha.value
autoproc.refinedCell_beta = xdsout.cell_beta.value
autoproc.refinedCell_gamma = xdsout.cell_gamma.value
output.AutoProc = autoproc
# scaling container and all the things that go in
scaling_container_noanom = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
scaling_container_noanom.AutoProcScaling = scaling
# NOANOM PATH
xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged
inner_stats_noanom = xscale_stats_noanom.completeness_entries[0]
outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1]
# use the previous shell's res as low res, if available
prev_res = self.low_resolution_limit
try:
prev_res = xscale_stats_noanom.completeness_entries[-2].outer_res.value
except IndexError:
pass
total_stats_noanom = xscale_stats_noanom.total_completeness
stats = _create_scaling_stats(inner_stats_noanom, 'innerShell',
self.low_resolution_limit, False)
overall_low = stats.resolutionLimitLow
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(outer_stats_noanom, 'outerShell',
prev_res, False)
overall_high = stats.resolutionLimitHigh
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(total_stats_noanom, 'overall',
self.low_resolution_limit, False)
stats.resolutionLimitLow = overall_low
stats.resolutionLimitHigh = overall_high
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
integration_container_noanom = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.data_collection_id.value
integration_container_noanom.Image = image
integration_noanom = AutoProcIntegration()
if self.integration_id_noanom is not None:
integration_noanom.autoProcIntegrationId = self.integration_id_noanom
crystal_stats = self.parse_xds_noanom.dataOutput
integration_noanom.cell_a = crystal_stats.cell_a.value
integration_noanom.cell_b = crystal_stats.cell_b.value
integration_noanom.cell_c = crystal_stats.cell_c.value
integration_noanom.cell_alpha = crystal_stats.cell_alpha.value
integration_noanom.cell_beta = crystal_stats.cell_beta.value
integration_noanom.cell_gamma = crystal_stats.cell_gamma.value
integration_noanom.anomalous = 0
# done with the integration
integration_container_noanom.AutoProcIntegration = integration_noanom
scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom
# ANOM PATH
scaling_container_anom = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
scaling_container_anom.AutoProcScaling = scaling
xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged
inner_stats_anom = xscale_stats_anom.completeness_entries[0]
outer_stats_anom = xscale_stats_anom.completeness_entries[-1]
# use the previous shell's res as low res if available
prev_res = self.low_resolution_limit
try:
prev_res = xscale_stats_anom.completeness_entries[-2].outer_res.value
except IndexError:
pass
total_stats_anom = xscale_stats_anom.total_completeness
stats = _create_scaling_stats(inner_stats_anom, 'innerShell',
self.low_resolution_limit, True)
overall_low = stats.resolutionLimitLow
scaling_container_anom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(outer_stats_anom, 'outerShell',
prev_res, True)
overall_high = stats.resolutionLimitHigh
scaling_container_anom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(total_stats_anom, 'overall',
self.low_resolution_limit, True)
stats.resolutionLimitLow = overall_low
stats.resolutionLimitHigh = overall_high
scaling_container_anom.AutoProcScalingStatistics.append(stats)
integration_container_anom = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.data_collection_id.value
integration_container_anom.Image = image
integration_anom = AutoProcIntegration()
crystal_stats = self.parse_xds_anom.dataOutput
if self.integration_id_anom is not None:
integration_anom.autoProcIntegrationId = self.integration_id_anom
integration_anom.cell_a = crystal_stats.cell_a.value
integration_anom.cell_b = crystal_stats.cell_b.value
integration_anom.cell_c = crystal_stats.cell_c.value
integration_anom.cell_alpha = crystal_stats.cell_alpha.value
integration_anom.cell_beta = crystal_stats.cell_beta.value
integration_anom.cell_gamma = crystal_stats.cell_gamma.value
integration_anom.anomalous = 1
# done with the integration
integration_container_anom.AutoProcIntegration = integration_anom
scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom
# ------ NO ANOM / ANOM end
program_container = AutoProcProgramContainer()
program_container.AutoProcProgram = AutoProcProgram()
program_container.AutoProcProgram.processingCommandLine = ' '.join(sys.argv)
program_container.AutoProcProgram.processingPrograms = 'edna-fastproc'
# now for the generated files. There's some magic to do with
# their paths to determine where to put them on pyarch
pyarch_path = None
# Note: the path is in the form /data/whatever
# remove the edna-autoproc-import suffix
original_files_dir = self.file_conversion.dataInput.output_directory.value
#files_dir, _ = os.path.split(original_files_dir)
files_dir = original_files_dir
# the whole transformation is fragile!
if files_dir.startswith('/data/visitor'):
# We might get empty elements at the head/tail of the list
tokens = [elem for elem in files_dir.split(os.path.sep)
if len(elem) > 0]
pyarch_path = os.path.join('/data/pyarch',
tokens[3], tokens[2],
*tokens[4:])
else:
# We might get empty elements at the head/tail of the list
tokens = [elem for elem in files_dir.split(os.path.sep)
if len(elem) > 0]
if tokens[2] == 'inhouse':
pyarch_path = os.path.join('/data/pyarch', tokens[1],
*tokens[3:])
if pyarch_path is not None:
pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA')
try:
os.makedirs(pyarch_path)
except OSError:
# dir already exists, may happen when testing
EDVerbose.screen('Target directory on pyarch ({0}) already exists, ignoring'.format(pyarch_path))
file_list = []
# we can now copy the files to this dir
for f in os.listdir(original_files_dir):
current = os.path.join(original_files_dir, f)
if not os.path.isfile(current):
continue
if not os.path.splitext(current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS:
continue
new_path = os.path.join(pyarch_path, f)
file_list.append(new_path)
shutil.copyfile(current,
new_path)
# now add those to the ispyb upload
for path in file_list:
dirname, filename = os.path.split(path)
attach = AutoProcProgramAttachment()
attach.fileType = "Result"
attach.fileName = filename
attach.filePath = dirname
program_container.AutoProcProgramAttachment.append(attach)
program_container.AutoProcProgram.processingStatus = True
output.AutoProcProgramContainer = program_container
# first with anom
output.AutoProcScalingContainer = scaling_container_anom
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
with open(self.dataInput.output_file.path.value, 'w') as f:
f.write(ispyb_input.marshal())
# store results in ispyb
self.store_autoproc_anom.dataInput = ispyb_input
t0=time.time()
self.store_autoproc_anom.executeSynchronous()
self.stats['ispyb_upload'] = time.time() - t0
with open(self.log_file_path, 'w') as f:
json.dump(self.stats, f)
if self.store_autoproc_anom.isFailure():
self.ERROR('could not send results to ispyb')
else:
# store the autoproc ID as a filename in the
# fastproc_integration_ids directory
os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755)
# then noanom stats
output.AutoProcScalingContainer = scaling_container_noanom
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
with open(self.dataInput.output_file.path.value, 'w') as f:
f.write(ispyb_input.marshal())
# store results in ispyb
self.store_autoproc_noanom.dataInput = ispyb_input
t0=time.time()
self.store_autoproc_noanom.executeSynchronous()
self.stats['ispyb_upload'] = time.time() - t0
with open(self.log_file_path, 'w') as f:
json.dump(self.stats, f)
if self.store_autoproc_noanom.isFailure():
self.ERROR('could not send results to ispyb')
else:
# store the autoproc id
os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlAutoproc.postProcess")
#Now that we have executed the whole thing we need to create
#the suitable ISPyB plugin input and serialize it to the file
#we've been given as input
output = AutoProcContainer()
# AutoProc attr
autoproc = AutoProc()
xdsout = self.xds_first.dataOutput
if xdsout.sg_number is not None: # and it should not
autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value]
autoproc.refinedCell_a = xdsout.cell_a.value
autoproc.refinedCell_b = xdsout.cell_b.value
autoproc.refinedCell_c = xdsout.cell_c.value
autoproc.refinedCell_alpha = xdsout.cell_alpha.value
autoproc.refinedCell_beta = xdsout.cell_beta.value
autoproc.refinedCell_gamma = xdsout.cell_gamma.value
output.AutoProc = autoproc
# scaling container and all the things that go in
scaling_container_noanom = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
scaling_container_noanom.AutoProcScaling = scaling
# NOANOM PATH
xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged
inner_stats_noanom = xscale_stats_noanom.completeness_entries[0]
outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1]
# use the previous shell's res as low res, if available
prev_res = self.low_resolution_limit
try:
prev_res = xscale_stats_noanom.completeness_entries[
-2].outer_res.value
except IndexError:
pass
total_stats_noanom = xscale_stats_noanom.total_completeness
stats = _create_scaling_stats(inner_stats_noanom, 'innerShell',
self.low_resolution_limit, False)
overall_low = stats.resolutionLimitLow
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(outer_stats_noanom, 'outerShell',
prev_res, False)
overall_high = stats.resolutionLimitHigh
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(total_stats_noanom, 'overall',
self.low_resolution_limit, False)
stats.resolutionLimitLow = overall_low
stats.resolutionLimitHigh = overall_high
scaling_container_noanom.AutoProcScalingStatistics.append(stats)
integration_container_noanom = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.data_collection_id.value
integration_container_noanom.Image = image
integration_noanom = AutoProcIntegration()
if self.integration_id_noanom is not None:
integration_noanom.autoProcIntegrationId = self.integration_id_noanom
crystal_stats = self.parse_xds_noanom.dataOutput
integration_noanom.cell_a = crystal_stats.cell_a.value
integration_noanom.cell_b = crystal_stats.cell_b.value
integration_noanom.cell_c = crystal_stats.cell_c.value
integration_noanom.cell_alpha = crystal_stats.cell_alpha.value
integration_noanom.cell_beta = crystal_stats.cell_beta.value
integration_noanom.cell_gamma = crystal_stats.cell_gamma.value
integration_noanom.anomalous = 0
# done with the integration
integration_container_noanom.AutoProcIntegration = integration_noanom
scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom
# ANOM PATH
scaling_container_anom = AutoProcScalingContainer()
scaling = AutoProcScaling()
scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime())
scaling_container_anom.AutoProcScaling = scaling
xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged
inner_stats_anom = xscale_stats_anom.completeness_entries[0]
outer_stats_anom = xscale_stats_anom.completeness_entries[-1]
# use the previous shell's res as low res if available
prev_res = self.low_resolution_limit
try:
prev_res = xscale_stats_anom.completeness_entries[
-2].outer_res.value
except IndexError:
pass
total_stats_anom = xscale_stats_anom.total_completeness
stats = _create_scaling_stats(inner_stats_anom, 'innerShell',
self.low_resolution_limit, True)
overall_low = stats.resolutionLimitLow
scaling_container_anom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res,
True)
overall_high = stats.resolutionLimitHigh
scaling_container_anom.AutoProcScalingStatistics.append(stats)
stats = _create_scaling_stats(total_stats_anom, 'overall',
self.low_resolution_limit, True)
stats.resolutionLimitLow = overall_low
stats.resolutionLimitHigh = overall_high
scaling_container_anom.AutoProcScalingStatistics.append(stats)
integration_container_anom = AutoProcIntegrationContainer()
image = Image()
image.dataCollectionId = self.dataInput.data_collection_id.value
integration_container_anom.Image = image
integration_anom = AutoProcIntegration()
crystal_stats = self.parse_xds_anom.dataOutput
if self.integration_id_anom is not None:
integration_anom.autoProcIntegrationId = self.integration_id_anom
integration_anom.cell_a = crystal_stats.cell_a.value
integration_anom.cell_b = crystal_stats.cell_b.value
integration_anom.cell_c = crystal_stats.cell_c.value
integration_anom.cell_alpha = crystal_stats.cell_alpha.value
integration_anom.cell_beta = crystal_stats.cell_beta.value
integration_anom.cell_gamma = crystal_stats.cell_gamma.value
integration_anom.anomalous = 1
# done with the integration
integration_container_anom.AutoProcIntegration = integration_anom
scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom
# ------ NO ANOM / ANOM end
program_container = AutoProcProgramContainer()
program_container.AutoProcProgram = AutoProcProgram()
program_container.AutoProcProgram.processingCommandLine = ' '.join(
sys.argv)
program_container.AutoProcProgram.processingPrograms = 'edna-fastproc'
# now for the generated files. There's some magic to do with
# their paths to determine where to put them on pyarch
pyarch_path = None
# Note: the path is in the form /data/whatever
# remove the edna-autoproc-import suffix
original_files_dir = self.file_conversion.dataInput.output_directory.value
#files_dir, _ = os.path.split(original_files_dir)
files_dir = original_files_dir
# the whole transformation is fragile!
if files_dir.startswith('/data/visitor'):
# We might get empty elements at the head/tail of the list
tokens = [
elem for elem in files_dir.split(os.path.sep) if len(elem) > 0
]
pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2],
*tokens[4:])
else:
# We might get empty elements at the head/tail of the list
tokens = [
elem for elem in files_dir.split(os.path.sep) if len(elem) > 0
]
if tokens[2] == 'inhouse':
pyarch_path = os.path.join('/data/pyarch', tokens[1],
*tokens[3:])
if pyarch_path is not None:
pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA')
try:
os.makedirs(pyarch_path)
except OSError:
# dir already exists, may happen when testing
EDVerbose.screen(
'Target directory on pyarch ({0}) already exists, ignoring'
.format(pyarch_path))
file_list = []
# we can now copy the files to this dir
for f in os.listdir(original_files_dir):
current = os.path.join(original_files_dir, f)
if not os.path.isfile(current):
continue
if not os.path.splitext(
current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS:
continue
new_path = os.path.join(pyarch_path, f)
file_list.append(new_path)
shutil.copyfile(current, new_path)
# now add those to the ispyb upload
for path in file_list:
dirname, filename = os.path.split(path)
attach = AutoProcProgramAttachment()
attach.fileType = "Result"
attach.fileName = filename
attach.filePath = dirname
program_container.AutoProcProgramAttachment.append(attach)
program_container.AutoProcProgram.processingStatus = True
output.AutoProcProgramContainer = program_container
# first with anom
output.AutoProcScalingContainer = scaling_container_anom
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
with open(self.dataInput.output_file.path.value, 'w') as f:
f.write(ispyb_input.marshal())
# store results in ispyb
self.store_autoproc_anom.dataInput = ispyb_input
t0 = time.time()
self.store_autoproc_anom.executeSynchronous()
self.stats['ispyb_upload'] = time.time() - t0
with open(self.log_file_path, 'w') as f:
json.dump(self.stats, f)
if self.store_autoproc_anom.isFailure():
self.ERROR('could not send results to ispyb')
else:
# store the autoproc ID as a filename in the
# fastproc_integration_ids directory
os.mknod(
os.path.join(self.autoproc_ids_dir,
str(self.integration_id_anom)), 0755)
# then noanom stats
output.AutoProcScalingContainer = scaling_container_noanom
ispyb_input = XSDataInputStoreAutoProc()
ispyb_input.AutoProcContainer = output
with open(self.dataInput.output_file.path.value, 'w') as f:
f.write(ispyb_input.marshal())
# store results in ispyb
self.store_autoproc_noanom.dataInput = ispyb_input
t0 = time.time()
self.store_autoproc_noanom.executeSynchronous()
self.stats['ispyb_upload'] = time.time() - t0
with open(self.log_file_path, 'w') as f:
json.dump(self.stats, f)
if self.store_autoproc_noanom.isFailure():
self.ERROR('could not send results to ispyb')
else:
# store the autoproc id
os.mknod(
os.path.join(self.autoproc_ids_dir,
str(self.integration_id_noanom)), 0755)
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
if isStaraniso:
staranisoString = "_staraniso"
else:
staranisoString = ""
# Read the generated ISPyB xml file
pathToISPyBXML = None
if isStaraniso:
if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
if pathToISPyBXML is not None:
autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
if self.dataInput.dataCollectionId is not None:
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
if isAnom:
autoProcIntegration.anomalous = True
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
else:
autoProcIntegration.anomalous = False
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
autoProcProgram.processingPrograms = "autoPROC" + staranisoString
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgram.processingStatus = "SUCCESS"
# EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
# Check if summary_inlined.html exists
summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
if os.path.exists(summaryInlinedHtmlPath):
summaryName = "summary_inlined"
summaryHtmlPath = summaryInlinedHtmlPath
else:
summaryName = "summary"
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Upload summary.html
pathtoFile = summaryHtmlPath
pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
if not pyarchFile in self.listPyarchFile:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
self.listPyarchFile.append(pyarchFile)
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
if summaryName == "summary":
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
if self.pyarchDirectory is not None:
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
autoProcProgramAttachmentPdf.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
if self.pyarchDirectory is not None:
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
f_in = open(pathToXdsAsciiHkl)
f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
f_out.writelines(f_in)
f_out.close()
f_in.close()
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add report.pdf
pathToRepordPdf = None
if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
if pathToRepordPdf is not None:
pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchReportFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoprocAnom.executeSynchronous()
isSuccess = not edPluginStoreAutoprocAnom.isFailure()
if isSuccess:
self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
if isAnom:
if isStaraniso:
self.hasUploadedAnomStaranisoResultsToISPyB = True
else:
self.hasUploadedAnomResultsToISPyB = True
else:
if isStaraniso:
self.hasUploadedNoanomStaranisoResultsToISPyB = True
else:
self.hasUploadedNoanomResultsToISPyB = True
else:
self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))