def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = trunc + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
# Read the generated ISPyB xml file - if any
if edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
autoProcProgram.processingPrograms = "XIA2_DIALS"
autoProcProgram.processingStatus = True
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.pdf"
# Convert the xia2.html to xia2.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(pathToHtmlFile)))
xsDataInputHTML2PDF.paperSize = XSDataString("A4")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
if pathToLogFile.endswith(".log"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId,
integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(
edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status="SUCCESS",
timeStart=self.timeStart,
timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToSummaryFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToHtmlFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToLogFile)
# Copy all log files to results:
shutil.copy(
pathToLogFile,
os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(
pathToLogFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
pathToDataFile)
shutil.copy(
pathToDataFile,
os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(
pathToDataFile,
os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(
autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin(
"EDPluginISPyBStoreAutoProcv1_4",
"EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(
trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value,
os.path.join(self.resultsDirectory, newFileName))
return successUpload
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId):
successUpload = False
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcIntegration.autoProcIntegrationId = integrationId
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
autoProcProgramContainer.AutoProcProgramAttachment = []
# Upload the log file to ISPyB
if edPluginExecXia2DIALS.dataOutput.logFile is not None:
pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the summary file to ISPyB
if edPluginExecXia2DIALS.dataOutput.summary is not None:
pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Create a pdf file of the html page
if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy all log files
for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
pathToLogFile = logFile.path.value
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
# Copy all log files to results:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
# Only copy .log files to pyarch
if pathToLogFile.endswith(".log"):
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Copy data files
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
pathToDataFile = dataFile.path.value
if pathToDataFile.endswith(".mtz"):
pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchFileName
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoproc.executeSynchronous()
successUpload = not edPluginStoreAutoproc.isFailure()
else:
# Copy dataFiles to results directory
for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
trunc, suffix = os.path.splitext(dataFile.path.value)
newFileName = os.path.basename(trunc) + "_" + anomString + suffix
shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
return successUpload
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
# Read the generated ISPyB xml file - if any
if edPluginExecAutoPROC.dataOutput.ispybXML is not None:
autoProcContainer = AutoProcContainer.parseFile(edPluginExecAutoPROC.dataOutput.ispybXML.path.value)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
if isAnom:
autoProcIntegration.anomalous = True
else:
autoProcIntegration.anomalous = False
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
autoProcProgram.processingPrograms = "autoPROC"
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachment.fileName = pyarchPdfFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "xscale_XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl):
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
f_in = open(pathToXdsAsciiHkl)
f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
f_out.writelines(f_in)
f_out.close()
f_in.close()
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
if isAnom:
self.edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
self.edPluginStoreAutoprocAnom.executeSynchronous()
else:
self.edPluginStoreAutoprocNoanom.dataInput = xsDataInputStoreAutoProc
self.edPluginStoreAutoprocNoanom.executeSynchronous()
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
if isStaraniso:
staranisoString = "_staraniso"
else:
staranisoString = ""
# Read the generated ISPyB xml file
pathToISPyBXML = None
if isStaraniso:
if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
if pathToISPyBXML is not None:
autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100
autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100
autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100
autoProcScalingStatistics.rPimAllIPlusIMinus *= 100
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
if self.dataInput.dataCollectionId is not None:
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
if isAnom:
autoProcIntegration.anomalous = True
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
else:
autoProcIntegration.anomalous = False
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
if self.reprocess:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess"
else:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgram.processingStatus = "SUCCESS"
# EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
# Check if summary_inlined.html exists
summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
if os.path.exists(summaryInlinedHtmlPath):
summaryName = "summary_inlined"
summaryHtmlPath = summaryInlinedHtmlPath
else:
summaryName = "summary"
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Upload summary.html
pathtoFile = summaryHtmlPath
pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
if not pyarchFile in self.listPyarchFile:
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
self.listPyarchFile.append(pyarchFile)
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
if summaryName == "summary":
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
if self.resultsDirectory:
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
if self.pyarchDirectory is not None:
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
autoProcProgramAttachmentPdf.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
if self.pyarchDirectory is not None:
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
with open(pathToXdsAsciiHkl, 'rb') as f_in:
with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
if self.resultsDirectory:
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add report.pdf
pathToRepordPdf = None
if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
if pathToRepordPdf is not None:
pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
if self.resultsDirectory:
shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchReportFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoprocAnom.executeSynchronous()
isSuccess = not edPluginStoreAutoprocAnom.isFailure()
if isSuccess:
self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
if isAnom:
if isStaraniso:
self.hasUploadedAnomStaranisoResultsToISPyB = True
else:
self.hasUploadedAnomResultsToISPyB = True
else:
if isStaraniso:
self.hasUploadedNoanomStaranisoResultsToISPyB = True
else:
self.hasUploadedNoanomResultsToISPyB = True
else:
self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
if isStaraniso:
staranisoString = "_staraniso"
else:
staranisoString = ""
# Read the generated ISPyB xml file
pathToISPyBXML = None
if isStaraniso:
if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
if pathToISPyBXML is not None:
autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
if self.dataInput.dataCollectionId is not None:
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
if isAnom:
autoProcIntegration.anomalous = True
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
else:
autoProcIntegration.anomalous = False
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
autoProcProgram.processingPrograms = "autoPROC" + staranisoString
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgram.processingStatus = "SUCCESS"
# EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
# Check if summary_inlined.html exists
summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
if os.path.exists(summaryInlinedHtmlPath):
summaryName = "summary_inlined"
summaryHtmlPath = summaryInlinedHtmlPath
else:
summaryName = "summary"
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Upload summary.html
pathtoFile = summaryHtmlPath
pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
if not pyarchFile in self.listPyarchFile:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
self.listPyarchFile.append(pyarchFile)
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
if summaryName == "summary":
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
if self.pyarchDirectory is not None:
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
autoProcProgramAttachmentPdf.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
if self.pyarchDirectory is not None:
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
f_in = open(pathToXdsAsciiHkl)
f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
f_out.writelines(f_in)
f_out.close()
f_in.close()
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add report.pdf
pathToRepordPdf = None
if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
if pathToRepordPdf is not None:
pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchReportFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoprocAnom.executeSynchronous()
isSuccess = not edPluginStoreAutoprocAnom.isFailure()
if isSuccess:
self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
if isAnom:
if isStaraniso:
self.hasUploadedAnomStaranisoResultsToISPyB = True
else:
self.hasUploadedAnomResultsToISPyB = True
else:
if isStaraniso:
self.hasUploadedNoanomStaranisoResultsToISPyB = True
else:
self.hasUploadedNoanomResultsToISPyB = True
else:
self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
