def create_integration_id(self, datacollect_id, comments): autoproc_status = edFactoryPlugin.loadPlugin( 'EDPluginISPyBStoreAutoProcStatusv1_4') status_input = XSDataInputStoreAutoProcStatus() status_input.dataCollectionId = datacollect_id status_input.anomalous = True # needed even if we only want to get an integration ID? status_data = AutoProcStatus() status_data.step = "Indexing" status_data.status = "Launched" status_data.comments = comments # Program autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=None, status="RUNNING", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) status_input.AutoProcProgram = autoProcProgram status_input.AutoProcStatus = status_data autoproc_status.dataInput = status_input # get our EDNAproc status id autoproc_status.executeSynchronous() return (autoproc_status.dataOutput.autoProcIntegrationId, autoproc_status.dataOutput.autoProcProgramId)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]: setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlCrystFELv1_0.postProcess") """ autoProcScalingStatisticsId : integer optional scalingStatisticsType : string comments : string resolutionLimitLow : float resolutionLimitHigh : float rMerge : float rMeasWithinIPlusIMinus : float rMeasAllIPlusIMinus : float rPimWithinIPlusIMinus : float rPimAllIPlusIMinus : float fractionalPartialBias : float nTotalObservations : integer ntotalUniqueObservations : integer meanIOverSigI : float completeness : float multiplicity : float anomalousCompleteness : float anomalousMultiplicity : float recordTimeStamp : string anomalous : boolean autoProcScalingId : integer ccHalf : float ccAno : float sigAno : float isa : float completenessSpherical : float anomalousCompletenessSpherical : float completenessEllipsoidal : float anomalousCompletenessEllipsoidal : float """ """ overallCompl : XSDataDouble overallRed : XSDataDouble overallSnr : XSDataDouble overallRsplit : XSDataDouble overallCC """ output = AutoProcContainer() autoproc = AutoProc() autoproc.spaceGroup = str(self.dataInput.spaceGroup.value) output.AutoProc = autoproc scaling_container = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container.AutoProcScaling = scaling #Open the cell file and get the cell parameters cell_file = open(self.dataInput.cellFile.value, "r") cell_params = {} for line in cell_file.readlines(): if line.startswith("a ="): cell_params["a"] = float(line[4:9]) elif line.startswith("b ="): cell_params["b"] = float(line[4:9]) elif line.startswith("c ="): cell_params["c"] = float(line[4:9]) elif line.startswith("al ="): cell_params["alpha"] = float(line[4:9]) elif line.startswith("be ="): cell_params["beta"] = float(line[4:9]) elif line.startswith("ga ="): cell_params["gamma"] = float(line[4:9]) cell_file.close() autoproc.refinedCell_a = cell_params["a"] autoproc.refinedCell_b = cell_params["b"] autoproc.refinedCell_c = cell_params["c"] autoproc.refinedCell_alpha = cell_params["alpha"] autoproc.refinedCell_beta = cell_params["beta"] autoproc.refinedCell_gamma = cell_params["gamma"] overall_stats = AutoProcScalingStatistics() overall_stats.scalingStatisticsType = "innerShell" overall_stats.ccHalf = float( self.post_process_plugin.dataOutput.overallCC.value) overall_stats.resolutionLimitLow = float( self.post_process_plugin.dataOutput.resolutionLimitLow.value) overall_stats.resolutionLimitHigh = float( self.post_process_plugin.dataOutput.resolutionLimitHigh.value) overall_stats.completeness = float( self.post_process_plugin.dataOutput.overallCompl.value) #TODO get rMerge: otherwise results are not displayed in EXI overall_stats.rMerge = 0 scaling_container.AutoProcScalingStatistics.append(overall_stats) integration_container = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.dataCollectionId.value integration_container.Image = image integration = AutoProcIntegration() if self.integration_id is not None: integration.autoProcIntegrationId = self.integration_id integration.cell_a = cell_params["a"] integration.cell_b = cell_params["b"] integration.cell_c = cell_params["c"] integration.cell_alpha = cell_params["alpha"] integration.cell_beta = cell_params["beta"] integration.cell_gamma = cell_params["gamma"] integration.anomalous = 0 # done with the integration integration_container.AutoProcIntegration = integration scaling_container.AutoProcIntegrationContainer = integration_container program_container = AutoProcProgramContainer() program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=self.program_id, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) output.AutoProcProgramContainer = program_container output.AutoProcScalingContainer = scaling_container for log_file in self.post_process_plugin.dataOutput.logFiles: pathToLogFile = log_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToLogFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Log" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) for data_file in self.post_process_plugin.dataOutput.dataFiles: pathToDataFile = data_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToDataFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Result" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output self.store_autoproc_plugin.dataInput = ispyb_input self.store_autoproc_plugin.executeSynchronous() if self.store_autoproc_plugin.isFailure(): self.ERROR("Could not upload results to ispyb!") else: self.hasUploadedAnomResultsToISPyB = True self.screen("Results uploaded to ISPyB") xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment() xsDataInput.newComment = XSDataString( self.post_process_plugin.dataOutput.comment.value) xsDataInput.dataCollectionId = self.dataInput.dataCollectionId self.store_dc_comment_plugin.dataInput = xsDataInput self.executePluginSynchronous(self.store_dc_comment_plugin)
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile( edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToLogFile) # Copy all log files to results: shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy( pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToDataFile) shutil.copy( pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin( "EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename( trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0o755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]: setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # Attached files pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory) pyarchResultPath = os.path.join(pyarchPath, "results") if not os.path.exists(pyarchResultPath): os.makedirs(pyarchResultPath, 0o755) # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile) # Copy all log files to results: shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile) shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename(trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def testCreateAutoProcProgram(self): autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram() print(autoProcProgram.marshal())
def testCreateAutoProcProgram(self): autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram() print(autoProcProgram.marshal())