def addAttachment(self, autoProcProgramContainer, strPath, name, suffix, isAnom=True, isMerged=None, attachmentType="Log", doGzip=False, noMergedString=False): if isAnom: anomString = "_anom" else: anomString = "_noanom" if noMergedString: pyarchFileName = self.pyarchPrefix + anomString + "_{0}.{1}".format(name, suffix) else: if isMerged is not None: if isMerged: mergeString = "_merged" else: mergeString = "_unmerged" else: mergeString = "" pyarchFileName = self.pyarchPrefix + mergeString + anomString + "_{0}.{1}".format(name, suffix) shutil.copy(strPath, os.path.join(self.resultsDirectory, pyarchFileName)) if doGzip: pyarchFileName += ".gz" f_in = open(strPath) f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchFileName), "wb") f_out.writelines(f_in) f_out.close() f_in.close() else: shutil.copy(strPath, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = attachmentType autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
def addAttachment(self, autoProcProgramContainer, strPath, name, suffix, isAnom=True, isMerged=None, attachmentType="Log", doGzip=False, noMergedString=False): if isAnom: anomString = "_anom" else: anomString = "_noanom" if noMergedString: pyarchFileName = self.pyarchPrefix + anomString + "_{0}.{1}".format(name, suffix) else: if isMerged is not None: if isMerged: mergeString = "_merged" else: mergeString = "_unmerged" else: mergeString = "" pyarchFileName = self.pyarchPrefix + mergeString + anomString + "_{0}.{1}".format(name, suffix) shutil.copy(strPath, os.path.join(self.resultsDirectory, pyarchFileName)) if doGzip: pyarchFileName += ".gz" with open(strPath, 'rb') as f_in: with gzip.open(os.path.join(self.pyarchDirectory, pyarchFileName), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) else: shutil.copy(strPath, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = attachmentType autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunDimplev1_0.process") if self.strPdbPath is None: self.screen("No pdb file found, not running dimple") return # Run dimple xsDataResultDimple = self.runDimple(self.strPdbPath, self.strPathNoanomAimlessMtz) if xsDataResultDimple is not None: # Create HTML page listHtmlPath = self.createHtmlPage( self.dataInput.imagePrefix.value, xsDataResultDimple, os.path.join(self.getWorkingDirectory(), "html"), self.dataInput.proposal.value, self.dataInput.sessionDate.value, self.dataInput.beamline.value, ) xsDataInputHTML2PDF = XSDataInputHTML2PDF() for strHtmlPath in listHtmlPath: xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(strHtmlPath))) xsDataInputHTML2PDF.resultDirectory = xsDataResultDimple.resultsDirectory edPluginHTML2PDF = self.loadPlugin("EDPluginHTML2PDFv1_0") edPluginHTML2PDF.dataInput = xsDataInputHTML2PDF edPluginHTML2PDF.executeSynchronous() strPdfFile = edPluginHTML2PDF.dataOutput.pdfFile.path.value # Copy result files listOfTargetPaths = self.copyResults( self.dataInput.imagePrefix.value, self.strPathToResults, xsDataResultDimple, strPdfFile ) # Upload files to ISPyB if self.dataInput.autoProcProgramId is not None and self.dataInput.pyarchPath is not None: strPyarchRootPath = self.dataInput.pyarchPath.path.value xsDataInputStoreAutoProcProgramAttachment = XSDataInputStoreAutoProcProgramAttachment() for targetPath in listOfTargetPaths: shutil.copy(targetPath, strPyarchRootPath) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileType = "Result" autoProcProgramAttachment.fileName = os.path.basename(targetPath) autoProcProgramAttachment.filePath = strPyarchRootPath autoProcProgramAttachment.autoProcProgramId = self.dataInput.autoProcProgramId.value xsDataInputStoreAutoProcProgramAttachment.addAutoProcProgramAttachment(autoProcProgramAttachment) edPluginStoreAutoProcProgramAttachment = self.loadPlugin( "EDPluginISPyBStoreAutoProcProgramAttachmentv1_4" ) edPluginStoreAutoProcProgramAttachment.dataInput = xsDataInputStoreAutoProcProgramAttachment edPluginStoreAutoProcProgramAttachment.executeSynchronous()
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = trunc + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) # Read the generated ISPyB xml file - if any if edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram autoProcProgram.processingPrograms = "XIA2_DIALS" autoProcProgram.processingStatus = True autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.pdf" # Convert the xia2.html to xia2.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(pathToHtmlFile))) xsDataInputHTML2PDF.paperSize = XSDataString("A4") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value if pathToLogFile.endswith(".log"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile) shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile) shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous()
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlCrystFELv1_0.postProcess") """ autoProcScalingStatisticsId : integer optional scalingStatisticsType : string comments : string resolutionLimitLow : float resolutionLimitHigh : float rMerge : float rMeasWithinIPlusIMinus : float rMeasAllIPlusIMinus : float rPimWithinIPlusIMinus : float rPimAllIPlusIMinus : float fractionalPartialBias : float nTotalObservations : integer ntotalUniqueObservations : integer meanIOverSigI : float completeness : float multiplicity : float anomalousCompleteness : float anomalousMultiplicity : float recordTimeStamp : string anomalous : boolean autoProcScalingId : integer ccHalf : float ccAno : float sigAno : float isa : float completenessSpherical : float anomalousCompletenessSpherical : float completenessEllipsoidal : float anomalousCompletenessEllipsoidal : float """ """ overallCompl : XSDataDouble overallRed : XSDataDouble overallSnr : XSDataDouble overallRsplit : XSDataDouble overallCC """ output = AutoProcContainer() autoproc = AutoProc() autoproc.spaceGroup = str(self.dataInput.spaceGroup.value) output.AutoProc = autoproc scaling_container = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container.AutoProcScaling = scaling #Open the cell file and get the cell parameters cell_file = open(self.dataInput.cellFile.value, "r") cell_params = {} for line in cell_file.readlines(): if line.startswith("a ="): cell_params["a"] = float(line[4:9]) elif line.startswith("b ="): cell_params["b"] = float(line[4:9]) elif line.startswith("c ="): cell_params["c"] = float(line[4:9]) elif line.startswith("al ="): cell_params["alpha"] = float(line[4:9]) elif line.startswith("be ="): cell_params["beta"] = float(line[4:9]) elif line.startswith("ga ="): cell_params["gamma"] = float(line[4:9]) cell_file.close() autoproc.refinedCell_a = cell_params["a"] autoproc.refinedCell_b = cell_params["b"] autoproc.refinedCell_c = cell_params["c"] autoproc.refinedCell_alpha = cell_params["alpha"] autoproc.refinedCell_beta = cell_params["beta"] autoproc.refinedCell_gamma = cell_params["gamma"] overall_stats = AutoProcScalingStatistics() overall_stats.scalingStatisticsType = "innerShell" overall_stats.ccHalf = float( self.post_process_plugin.dataOutput.overallCC.value) overall_stats.resolutionLimitLow = float( self.post_process_plugin.dataOutput.resolutionLimitLow.value) overall_stats.resolutionLimitHigh = float( self.post_process_plugin.dataOutput.resolutionLimitHigh.value) overall_stats.completeness = float( self.post_process_plugin.dataOutput.overallCompl.value) #TODO get rMerge: otherwise results are not displayed in EXI overall_stats.rMerge = 0 scaling_container.AutoProcScalingStatistics.append(overall_stats) integration_container = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.dataCollectionId.value integration_container.Image = image integration = AutoProcIntegration() if self.integration_id is not None: integration.autoProcIntegrationId = self.integration_id integration.cell_a = cell_params["a"] integration.cell_b = cell_params["b"] integration.cell_c = cell_params["c"] integration.cell_alpha = cell_params["alpha"] integration.cell_beta = cell_params["beta"] integration.cell_gamma = cell_params["gamma"] integration.anomalous = 0 # done with the integration integration_container.AutoProcIntegration = integration scaling_container.AutoProcIntegrationContainer = integration_container program_container = AutoProcProgramContainer() program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=self.program_id, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) output.AutoProcProgramContainer = program_container output.AutoProcScalingContainer = scaling_container for log_file in self.post_process_plugin.dataOutput.logFiles: pathToLogFile = log_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToLogFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Log" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) for data_file in self.post_process_plugin.dataOutput.dataFiles: pathToDataFile = data_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToDataFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Result" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output self.store_autoproc_plugin.dataInput = ispyb_input self.store_autoproc_plugin.executeSynchronous() if self.store_autoproc_plugin.isFailure(): self.ERROR("Could not upload results to ispyb!") else: self.hasUploadedAnomResultsToISPyB = True self.screen("Results uploaded to ISPyB") xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment() xsDataInput.newComment = XSDataString( self.post_process_plugin.dataOutput.comment.value) xsDataInput.dataCollectionId = self.dataInput.dataCollectionId self.store_dc_comment_plugin.dataInput = xsDataInput self.executePluginSynchronous(self.store_dc_comment_plugin)
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile( edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToLogFile) # Copy all log files to results: shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy( pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToDataFile) shutil.copy( pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin( "EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename( trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunDimplev1_0.process") if self.strPdbPath is None: self.screen('No pdb file found, not running dimple') return # Run dimple xsDataResultDimple = self.runDimple(self.strPdbPath, self.strPathNoanomAimlessMtz) if xsDataResultDimple is not None: if self.dataInput.resultsDirectory is not None: # Copy results to PROCESSED_DATA strOrigDimpleResultsDirectory = xsDataResultDimple.resultsDirectory.path.value strResultsDirectory = self.dataInput.resultsDirectory.path.value targetDirectoryName = os.path.basename(strOrigDimpleResultsDirectory) if os.path.exists(os.path.join(strResultsDirectory, targetDirectoryName)): index = 1 newTargetDirectoryName = "{0}_{1:02d}".format(targetDirectoryName, index) while os.path.exists(os.path.join(strResultsDirectory, newTargetDirectoryName)): index += 1 newTargetDirectoryName = "{0}_{1:02d}".format(targetDirectoryName, index) targetDirectoryName = newTargetDirectoryName strDimpleResultsDirectory = os.path.join(strResultsDirectory, targetDirectoryName) shutil.copytree(strOrigDimpleResultsDirectory, strDimpleResultsDirectory) strDimpleHtmlPath = os.path.join(strDimpleResultsDirectory, "html") else: strDimpleHtmlPath = os.path.join(self.getWorkingDirectory(), "html") strDimpleResultsDirectory = xsDataResultDimple.resultsDirectory.path.value # Create HTML page listHtmlPath = self.createHtmlPage(self.dataInput.imagePrefix.value, xsDataResultDimple, strDimpleResultsDirectory, strDimpleHtmlPath, self.dataInput.proposal.value, self.dataInput.sessionDate.value, self.dataInput.beamline.value) xsDataInputHTML2PDF = XSDataInputHTML2PDF() for strHtmlPath in listHtmlPath: xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(strHtmlPath))) xsDataInputHTML2PDF.resultDirectory = XSDataFile(XSDataString(strDimpleResultsDirectory)) edPluginHTML2PDF = self.loadPlugin("EDPluginHTML2PDFv1_0") edPluginHTML2PDF.dataInput = xsDataInputHTML2PDF edPluginHTML2PDF.executeSynchronous() strPdfFile = edPluginHTML2PDF.dataOutput.pdfFile.path.value # Copy result files listOfTargetPaths = self.copyResults(self.dataInput.imagePrefix.value, self.strPathToResults, xsDataResultDimple, strPdfFile) # Upload files to ISPyB if self.dataInput.autoProcProgramId is not None and self.dataInput.pyarchPath is not None: strPyarchRootPath = self.dataInput.pyarchPath.path.value xsDataInputStoreAutoProcProgramAttachment = XSDataInputStoreAutoProcProgramAttachment() for targetPath in listOfTargetPaths: shutil.copy(targetPath, strPyarchRootPath) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileType = "Result" autoProcProgramAttachment.fileName = os.path.basename(targetPath) autoProcProgramAttachment.filePath = strPyarchRootPath autoProcProgramAttachment.autoProcProgramId = self.dataInput.autoProcProgramId.value xsDataInputStoreAutoProcProgramAttachment.addAutoProcProgramAttachment(autoProcProgramAttachment) edPluginStoreAutoProcProgramAttachment = self.loadPlugin("EDPluginISPyBStoreAutoProcProgramAttachmentv1_4") edPluginStoreAutoProcProgramAttachment.dataInput = xsDataInputStoreAutoProcProgramAttachment edPluginStoreAutoProcProgramAttachment.executeSynchronous()
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile) # Copy all log files to results: shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile) shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename(trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if edPluginExecAutoPROC.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecAutoPROC.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram autoProcProgram.processingPrograms = "autoPROC" autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachment.fileName = pyarchPdfFile autoProcProgramAttachment.filePath = self.pyarchDirectory elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "xscale_XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl): pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) f_in = open(pathToXdsAsciiHkl) f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb") f_out.writelines(f_in) f_out.close() f_in.close() shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer if isAnom: self.edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc self.edPluginStoreAutoprocAnom.executeSynchronous() else: self.edPluginStoreAutoprocNoanom.dataInput = xsDataInputStoreAutoProc self.edPluginStoreAutoprocNoanom.executeSynchronous()
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" if isStaraniso: staranisoString = "_staraniso" else: staranisoString = "" # Read the generated ISPyB xml file pathToISPyBXML = None if isStaraniso: if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.ispybXML is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value if pathToISPyBXML is not None: autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100 autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100 autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100 autoProcScalingStatistics.rPimAllIPlusIMinus *= 100 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image if self.dataInput.dataCollectionId is not None: image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram if isAnom: autoProcIntegration.anomalous = True if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom else: autoProcIntegration.anomalous = False if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom if self.reprocess: autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess" else: autoProcProgram.processingPrograms = "autoPROC" + staranisoString autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgram.processingStatus = "SUCCESS" # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ] for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": # Check if summary_inlined.html exists summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html") if os.path.exists(summaryInlinedHtmlPath): summaryName = "summary_inlined" summaryHtmlPath = summaryInlinedHtmlPath else: summaryName = "summary" summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Upload summary.html pathtoFile = summaryHtmlPath pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName) if not pyarchFile in self.listPyarchFile: if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) self.listPyarchFile.append(pyarchFile) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = os.path.basename(pyarchFile) autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" if summaryName == "summary": # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch if self.resultsDirectory: shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) if self.pyarchDirectory is not None: shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachmentPdf = AutoProcProgramAttachment() autoProcProgramAttachmentPdf.fileName = pyarchPdfFile autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory autoProcProgramAttachmentPdf.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf) elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) if self.resultsDirectory: shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) if self.pyarchDirectory is not None: shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None: pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) with open(pathToXdsAsciiHkl, 'rb') as f_in: with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) if self.resultsDirectory: shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) if self.resultsDirectory: shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) if self.pyarchDirectory is not None: shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add report.pdf pathToRepordPdf = None if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.reportPdf is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value if pathToRepordPdf is not None: pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf)) if self.resultsDirectory: shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile)) if self.pyarchDirectory is not None: shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchReportFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString)) edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoprocAnom.executeSynchronous() isSuccess = not edPluginStoreAutoprocAnom.isFailure() if isSuccess: self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString)) if isAnom: if isStaraniso: self.hasUploadedAnomStaranisoResultsToISPyB = True else: self.hasUploadedAnomResultsToISPyB = True else: if isStaraniso: self.hasUploadedNoanomStaranisoResultsToISPyB = True else: self.hasUploadedNoanomResultsToISPyB = True else: self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
def postProcess(self, _edObject = None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlAutoproc.postProcess") #Now that we have executed the whole thing we need to create #the suitable ISPyB plugin input and serialize it to the file #we've been given as input output = AutoProcContainer() # AutoProc attr autoproc = AutoProc() xdsout = self.xds_first.dataOutput if xdsout.sg_number is not None: # and it should not autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value] autoproc.refinedCell_a = xdsout.cell_a.value autoproc.refinedCell_b = xdsout.cell_b.value autoproc.refinedCell_c = xdsout.cell_c.value autoproc.refinedCell_alpha = xdsout.cell_alpha.value autoproc.refinedCell_beta = xdsout.cell_beta.value autoproc.refinedCell_gamma = xdsout.cell_gamma.value output.AutoProc = autoproc # scaling container and all the things that go in scaling_container_noanom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_noanom.AutoProcScaling = scaling # NOANOM PATH xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged inner_stats_noanom = xscale_stats_noanom.completeness_entries[0] outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1] # use the previous shell's res as low res, if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_noanom.completeness_entries[-2].outer_res.value except IndexError: pass total_stats_noanom = xscale_stats_noanom.total_completeness stats = _create_scaling_stats(inner_stats_noanom, 'innerShell', self.low_resolution_limit, False) overall_low = stats.resolutionLimitLow scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_noanom, 'outerShell', prev_res, False) overall_high = stats.resolutionLimitHigh scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_noanom, 'overall', self.low_resolution_limit, False) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_noanom.AutoProcScalingStatistics.append(stats) integration_container_noanom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_noanom.Image = image integration_noanom = AutoProcIntegration() if self.integration_id_noanom is not None: integration_noanom.autoProcIntegrationId = self.integration_id_noanom crystal_stats = self.parse_xds_noanom.dataOutput integration_noanom.cell_a = crystal_stats.cell_a.value integration_noanom.cell_b = crystal_stats.cell_b.value integration_noanom.cell_c = crystal_stats.cell_c.value integration_noanom.cell_alpha = crystal_stats.cell_alpha.value integration_noanom.cell_beta = crystal_stats.cell_beta.value integration_noanom.cell_gamma = crystal_stats.cell_gamma.value integration_noanom.anomalous = 0 # done with the integration integration_container_noanom.AutoProcIntegration = integration_noanom scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom # ANOM PATH scaling_container_anom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_anom.AutoProcScaling = scaling xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged inner_stats_anom = xscale_stats_anom.completeness_entries[0] outer_stats_anom = xscale_stats_anom.completeness_entries[-1] # use the previous shell's res as low res if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_anom.completeness_entries[-2].outer_res.value except IndexError: pass total_stats_anom = xscale_stats_anom.total_completeness stats = _create_scaling_stats(inner_stats_anom, 'innerShell', self.low_resolution_limit, True) overall_low = stats.resolutionLimitLow scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res, True) overall_high = stats.resolutionLimitHigh scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_anom, 'overall', self.low_resolution_limit, True) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_anom.AutoProcScalingStatistics.append(stats) integration_container_anom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_anom.Image = image integration_anom = AutoProcIntegration() crystal_stats = self.parse_xds_anom.dataOutput if self.integration_id_anom is not None: integration_anom.autoProcIntegrationId = self.integration_id_anom integration_anom.cell_a = crystal_stats.cell_a.value integration_anom.cell_b = crystal_stats.cell_b.value integration_anom.cell_c = crystal_stats.cell_c.value integration_anom.cell_alpha = crystal_stats.cell_alpha.value integration_anom.cell_beta = crystal_stats.cell_beta.value integration_anom.cell_gamma = crystal_stats.cell_gamma.value integration_anom.anomalous = 1 # done with the integration integration_container_anom.AutoProcIntegration = integration_anom scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom # ------ NO ANOM / ANOM end program_container = AutoProcProgramContainer() program_container.AutoProcProgram = AutoProcProgram() program_container.AutoProcProgram.processingCommandLine = ' '.join(sys.argv) program_container.AutoProcProgram.processingPrograms = 'edna-fastproc' # now for the generated files. There's some magic to do with # their paths to determine where to put them on pyarch pyarch_path = None # Note: the path is in the form /data/whatever # remove the edna-autoproc-import suffix original_files_dir = self.file_conversion.dataInput.output_directory.value #files_dir, _ = os.path.split(original_files_dir) files_dir = original_files_dir # the whole transformation is fragile! if files_dir.startswith('/data/visitor'): # We might get empty elements at the head/tail of the list tokens = [elem for elem in files_dir.split(os.path.sep) if len(elem) > 0] pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2], *tokens[4:]) else: # We might get empty elements at the head/tail of the list tokens = [elem for elem in files_dir.split(os.path.sep) if len(elem) > 0] if tokens[2] == 'inhouse': pyarch_path = os.path.join('/data/pyarch', tokens[1], *tokens[3:]) if pyarch_path is not None: pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA') try: os.makedirs(pyarch_path) except OSError: # dir already exists, may happen when testing EDVerbose.screen('Target directory on pyarch ({0}) already exists, ignoring'.format(pyarch_path)) file_list = [] # we can now copy the files to this dir for f in os.listdir(original_files_dir): current = os.path.join(original_files_dir, f) if not os.path.isfile(current): continue if not os.path.splitext(current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS: continue new_path = os.path.join(pyarch_path, f) file_list.append(new_path) shutil.copyfile(current, new_path) # now add those to the ispyb upload for path in file_list: dirname, filename = os.path.split(path) attach = AutoProcProgramAttachment() attach.fileType = "Result" attach.fileName = filename attach.filePath = dirname program_container.AutoProcProgramAttachment.append(attach) program_container.AutoProcProgram.processingStatus = True output.AutoProcProgramContainer = program_container # first with anom output.AutoProcScalingContainer = scaling_container_anom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_anom.dataInput = ispyb_input t0=time.time() self.store_autoproc_anom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_anom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc ID as a filename in the # fastproc_integration_ids directory os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755) # then noanom stats output.AutoProcScalingContainer = scaling_container_noanom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_noanom.dataInput = ispyb_input t0=time.time() self.store_autoproc_noanom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_noanom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc id os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlAutoproc.postProcess") #Now that we have executed the whole thing we need to create #the suitable ISPyB plugin input and serialize it to the file #we've been given as input output = AutoProcContainer() # AutoProc attr autoproc = AutoProc() xdsout = self.xds_first.dataOutput if xdsout.sg_number is not None: # and it should not autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value] autoproc.refinedCell_a = xdsout.cell_a.value autoproc.refinedCell_b = xdsout.cell_b.value autoproc.refinedCell_c = xdsout.cell_c.value autoproc.refinedCell_alpha = xdsout.cell_alpha.value autoproc.refinedCell_beta = xdsout.cell_beta.value autoproc.refinedCell_gamma = xdsout.cell_gamma.value output.AutoProc = autoproc # scaling container and all the things that go in scaling_container_noanom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_noanom.AutoProcScaling = scaling # NOANOM PATH xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged inner_stats_noanom = xscale_stats_noanom.completeness_entries[0] outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1] # use the previous shell's res as low res, if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_noanom.completeness_entries[ -2].outer_res.value except IndexError: pass total_stats_noanom = xscale_stats_noanom.total_completeness stats = _create_scaling_stats(inner_stats_noanom, 'innerShell', self.low_resolution_limit, False) overall_low = stats.resolutionLimitLow scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_noanom, 'outerShell', prev_res, False) overall_high = stats.resolutionLimitHigh scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_noanom, 'overall', self.low_resolution_limit, False) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_noanom.AutoProcScalingStatistics.append(stats) integration_container_noanom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_noanom.Image = image integration_noanom = AutoProcIntegration() if self.integration_id_noanom is not None: integration_noanom.autoProcIntegrationId = self.integration_id_noanom crystal_stats = self.parse_xds_noanom.dataOutput integration_noanom.cell_a = crystal_stats.cell_a.value integration_noanom.cell_b = crystal_stats.cell_b.value integration_noanom.cell_c = crystal_stats.cell_c.value integration_noanom.cell_alpha = crystal_stats.cell_alpha.value integration_noanom.cell_beta = crystal_stats.cell_beta.value integration_noanom.cell_gamma = crystal_stats.cell_gamma.value integration_noanom.anomalous = 0 # done with the integration integration_container_noanom.AutoProcIntegration = integration_noanom scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom # ANOM PATH scaling_container_anom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_anom.AutoProcScaling = scaling xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged inner_stats_anom = xscale_stats_anom.completeness_entries[0] outer_stats_anom = xscale_stats_anom.completeness_entries[-1] # use the previous shell's res as low res if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_anom.completeness_entries[ -2].outer_res.value except IndexError: pass total_stats_anom = xscale_stats_anom.total_completeness stats = _create_scaling_stats(inner_stats_anom, 'innerShell', self.low_resolution_limit, True) overall_low = stats.resolutionLimitLow scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res, True) overall_high = stats.resolutionLimitHigh scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_anom, 'overall', self.low_resolution_limit, True) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_anom.AutoProcScalingStatistics.append(stats) integration_container_anom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_anom.Image = image integration_anom = AutoProcIntegration() crystal_stats = self.parse_xds_anom.dataOutput if self.integration_id_anom is not None: integration_anom.autoProcIntegrationId = self.integration_id_anom integration_anom.cell_a = crystal_stats.cell_a.value integration_anom.cell_b = crystal_stats.cell_b.value integration_anom.cell_c = crystal_stats.cell_c.value integration_anom.cell_alpha = crystal_stats.cell_alpha.value integration_anom.cell_beta = crystal_stats.cell_beta.value integration_anom.cell_gamma = crystal_stats.cell_gamma.value integration_anom.anomalous = 1 # done with the integration integration_container_anom.AutoProcIntegration = integration_anom scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom # ------ NO ANOM / ANOM end program_container = AutoProcProgramContainer() program_container.AutoProcProgram = AutoProcProgram() program_container.AutoProcProgram.processingCommandLine = ' '.join( sys.argv) program_container.AutoProcProgram.processingPrograms = 'edna-fastproc' # now for the generated files. There's some magic to do with # their paths to determine where to put them on pyarch pyarch_path = None # Note: the path is in the form /data/whatever # remove the edna-autoproc-import suffix original_files_dir = self.file_conversion.dataInput.output_directory.value #files_dir, _ = os.path.split(original_files_dir) files_dir = original_files_dir # the whole transformation is fragile! if files_dir.startswith('/data/visitor'): # We might get empty elements at the head/tail of the list tokens = [ elem for elem in files_dir.split(os.path.sep) if len(elem) > 0 ] pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2], *tokens[4:]) else: # We might get empty elements at the head/tail of the list tokens = [ elem for elem in files_dir.split(os.path.sep) if len(elem) > 0 ] if tokens[2] == 'inhouse': pyarch_path = os.path.join('/data/pyarch', tokens[1], *tokens[3:]) if pyarch_path is not None: pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA') try: os.makedirs(pyarch_path) except OSError: # dir already exists, may happen when testing EDVerbose.screen( 'Target directory on pyarch ({0}) already exists, ignoring' .format(pyarch_path)) file_list = [] # we can now copy the files to this dir for f in os.listdir(original_files_dir): current = os.path.join(original_files_dir, f) if not os.path.isfile(current): continue if not os.path.splitext( current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS: continue new_path = os.path.join(pyarch_path, f) file_list.append(new_path) shutil.copyfile(current, new_path) # now add those to the ispyb upload for path in file_list: dirname, filename = os.path.split(path) attach = AutoProcProgramAttachment() attach.fileType = "Result" attach.fileName = filename attach.filePath = dirname program_container.AutoProcProgramAttachment.append(attach) program_container.AutoProcProgram.processingStatus = True output.AutoProcProgramContainer = program_container # first with anom output.AutoProcScalingContainer = scaling_container_anom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_anom.dataInput = ispyb_input t0 = time.time() self.store_autoproc_anom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_anom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc ID as a filename in the # fastproc_integration_ids directory os.mknod( os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755) # then noanom stats output.AutoProcScalingContainer = scaling_container_noanom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_noanom.dataInput = ispyb_input t0 = time.time() self.store_autoproc_noanom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_noanom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc id os.mknod( os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" if isStaraniso: staranisoString = "_staraniso" else: staranisoString = "" # Read the generated ISPyB xml file pathToISPyBXML = None if isStaraniso: if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.ispybXML is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value if pathToISPyBXML is not None: autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image if self.dataInput.dataCollectionId is not None: image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram if isAnom: autoProcIntegration.anomalous = True if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom else: autoProcIntegration.anomalous = False if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom autoProcProgram.processingPrograms = "autoPROC" + staranisoString autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgram.processingStatus = "SUCCESS" # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ] for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": # Check if summary_inlined.html exists summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html") if os.path.exists(summaryInlinedHtmlPath): summaryName = "summary_inlined" summaryHtmlPath = summaryInlinedHtmlPath else: summaryName = "summary" summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Upload summary.html pathtoFile = summaryHtmlPath pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName) if not pyarchFile in self.listPyarchFile: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) self.listPyarchFile.append(pyarchFile) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = os.path.basename(pyarchFile) autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" if summaryName == "summary": # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) if self.pyarchDirectory is not None: shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachmentPdf = AutoProcProgramAttachment() autoProcProgramAttachmentPdf.fileName = pyarchPdfFile autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory autoProcProgramAttachmentPdf.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf) elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) if self.pyarchDirectory is not None: shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None: pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) f_in = open(pathToXdsAsciiHkl) f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb") f_out.writelines(f_in) f_out.close() f_in.close() shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) if self.pyarchDirectory is not None: shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add report.pdf pathToRepordPdf = None if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.reportPdf is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value if pathToRepordPdf is not None: pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf)) shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile)) if self.pyarchDirectory is not None: shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchReportFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString)) edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoprocAnom.executeSynchronous() isSuccess = not edPluginStoreAutoprocAnom.isFailure() if isSuccess: self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString)) if isAnom: if isStaraniso: self.hasUploadedAnomStaranisoResultsToISPyB = True else: self.hasUploadedAnomResultsToISPyB = True else: if isStaraniso: self.hasUploadedNoanomStaranisoResultsToISPyB = True else: self.hasUploadedNoanomResultsToISPyB = True else: self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))