def testCheckParameters(self): xsDataInput = XSDataInputDataver() xsDataInput.inputCurve = [XSDataFile()] xsDataInput.outputCurve = XSDataFile() edPluginExecDataver = self.createPlugin() edPluginExecDataver.setDataInput(xsDataInput) edPluginExecDataver.checkParameters()
def average_buffers(self): """ Average out all buffers """ self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer)) xsdIn = XSDataInputDataver(inputCurve=[XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer]) if self.dataInput.bufferCurve: xsdIn.outputCurve = self.dataInput.bufferCurve else: xsdIn.outputCurve = XSDataFile(XSDataString(self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer))) self.__edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver) self.__edPluginDatAver.dataInput = xsdIn self.__edPluginDatAver.connectSUCCESS(self.doSuccessDatAver) self.__edPluginDatAver.connectFAILURE(self.doFailureDatAver) self.__edPluginDatAver.executeSynchronous()
def average_buffers(self): """ Average out all buffers """ self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer)) xsdIn = XSDataInputDataver(inputCurve=[XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer]) if self.dataInput.bufferCurve: xsdIn.outputCurve = self.dataInput.bufferCurve else: xsdIn.outputCurve = XSDataFile(XSDataString(self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver_%04i.dat" % len(self.hplc_run.for_buffer))) self.edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver) self.edPluginDatAver.dataInput = xsdIn self.edPluginDatAver.connectSUCCESS(self.doSuccessDatAver) self.edPluginDatAver.connectFAILURE(self.doFailureDatAver) self.edPluginDatAver.executeSynchronous()
def processRun(self, run): run.dump_json() hdf5 = run.save_hdf5() self.json = os.path.splitext(hdf5)[0] + ".json" self.xsDataResult.hplcFile = XSDataFile(XSDataString(hdf5)) self.xsDataResult.hplcImage = XSDataFile(XSDataString(run.make_plot())) for group in run.analyse(): self.lstExecutiveSummary.append("Merging frames from %s to %s" % (group[0], group[-1])) xsdFrames = [ XSDataFile(XSDataString(run.frames[i].subtracted)) for i in group ] outname = os.path.splitext(run.frames[ group[0]].subtracted)[0] + "_aver_%s.dat" % group[-1] edpugin = self.loadPlugin(self.strControlledPluginDatAver) edpugin.dataInput = XSDataInputDataver(outputCurve=XSDataFile( XSDataString(outname)), inputCurve=xsdFrames) edpugin.connectSUCCESS(self.doSuccessDatAver) edpugin.connectFAILURE(self.doFailureDatAver) edpugin.execute() run.merge_curves.append(outname) run.merge_analysis[outname] = None run.merge_Rg[outname] = None run.merge_framesDIC[outname] = [group[0], group[-1]] # Append to hdf5 run.append_hdf5()
def processRun(self, run): # EDPluginBioSaxsHPLCv1_5.dictHPLC[self.runId].reset() for idx in run.frames: run.frames[idx].purge_memory() run.dump_json() hdf5 = run.save_hdf5(name=self.__class__.__name__) self.json = os.path.splitext(hdf5)[0] + ".json" self.xsDataResult.hplcFile = XSDataFile(XSDataString(hdf5)) self.xsDataResult.hplcImage = XSDataFile(XSDataString(run.make_plot())) peaks = [] try: peaks = run.analyse() except Exception as error: traceback.print_stack() self.ERROR("EDPluginBioSaxsFlushHPLCv1_5: Error in run.analyse() %s" % error) for group in peaks: self.lstExecutiveSummary.append("Merging frames from %s to %s" % (group[0], group[-1])) xsdFrames = [XSDataFile(XSDataString(run.frames[i].subtracted)) for i in group] outname = os.path.splitext(run.frames[group[0]].subtracted)[0] + "_aver_%s.dat" % group[-1] edpugin = self.loadPlugin(self.strControlledPluginDatAver) edpugin.dataInput = XSDataInputDataver(outputCurve=XSDataFile(XSDataString(outname)), inputCurve=xsdFrames) edpugin.connectSUCCESS(self.doSuccessDatAver) edpugin.connectFAILURE(self.doFailureDatAver) edpugin.execute() run.merge_curves.append(outname) run.merge_analysis[outname] = None run.merge_Rg[outname] = None run.merge_framesDIC[outname] = [group[0], group[-1]] # Append to hdf5 run.append_hdf5()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_3.process") xsdwf = XSDataInputWaitMultiFile(timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist"%dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist"%inp) time.sleep(1.0) shutil.copyfile(inp, dst) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None : for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None) : if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None) : if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(self.lstMerged) self.WARNING(self.strRadiationDamage) self.lstSummary.append("WARNING: " + self.strRadiationDamage) self.lstSummary.append("Merging files: " + " ".join([os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin(self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) #outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS(self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE(self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin(self.__strControlledPluginAutoSub) base = "_".join(os.path.basename(self.__class__.lastSample.path.value).split("_")[:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) xsd = XSDataInputAutoSub(sampleCurve=self.__class__.lastSample, buffers=[self.__class__.lastBuffer, self.dataInput.mergedCurve], subtractedCurve=XSDataFile(XSDataString(sub)) ) self.__edPluginExecAutoSub.setDataInput(xsd) self.__edPluginExecAutoSub.connectSUCCESS(self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE(self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve self.__class__.lastSample = None else: self.__class__.lastSample = self.dataInput.mergedCurve
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_1.process") xsdwf = XSDataInputWaitMultiFile(timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: shutil.copyfile(self.lstInput[0].path.value, self.dataInput.mergedCurve.path.value) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None : for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None) : if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None) : if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities])) if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(self.lstMerged) self.WARNING(self.strRadiationDamage) self.lstSummary.append("WARNING: " + self.strRadiationDamage) self.lstSummary.append("Merging files: " + " ".join([os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: shutil.copyfile(self.lstMerged[0].path.value, self.dataInput.mergedCurve.path.value) else: self.__edPluginExecDataver = self.loadPlugin(self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) #outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS(self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE(self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if self.isFailure(): return self.__edPluginExecAutoRg = self.loadPlugin(self.__strControlledPluginAutoRG) xsd = XSDataInputAutoRg(inputCurve=[self.dataInput.mergedCurve]) self.__edPluginExecAutoRg.setDataInput(xsd) self.__edPluginExecAutoRg.connectSUCCESS(self.doSuccessExecAutoRg) self.__edPluginExecAutoRg.connectFAILURE(self.doFailureExecAutoRg) self.__edPluginExecAutoRg.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginAutoSubv1_0.process") basename = os.path.splitext(os.path.basename(self.sampleCurve))[0] self.bestBuffer = os.path.join(self.outdir, "%s_bestbuffer.dat" % basename) self.averBuffer = os.path.join(self.outdir, "%s_averbuffer.dat" % basename) for idx, name in enumerate(self.buffers): copy(name, os.path.join(self.outdir, "%s_buf%i.dat" % (basename, idx))) if len(self.buffers) == 1: self.actualBestBuffer = self.buffers[0] else: self.__edPluginDataver.dataInput = XSDataInputDataver( inputCurve=self.dataInput.buffers, outputCurve=XSDataFile(XSDataString(self.averBuffer))) self.__edPluginDataver.connectSUCCESS(self.doSuccessExecDataver) self.__edPluginDataver.connectFAILURE(self.doFailureExecDataver) self.__edPluginDataver.execute() if len(self.buffers) == 2: edPluginDatcmp = self.loadPlugin(self.__strPluginDatcmp) edPluginDatcmp.dataInput = XSDataInputDatcmp( inputCurve=self.dataInput.buffers) edPluginDatcmp.connectSUCCESS(self.doSuccessExecDatcmp) edPluginDatcmp.connectFAILURE(self.doFailureExecDatcmp) edPluginDatcmp.executeSynchronous() if self.isFailure() or (self.fidelity is None): return if self.fidelity < self.BUFFER_SIMILARITY: #buffer are not the same: keeping the one with lowest Rg/I0 edpluginRg = self.loadPlugin(self.__strPluginAutoRg) edpluginRg.dataInput = XSDataInputAutoRg( inputCurve=self.dataInput.buffers) edpluginRg.connectSUCCESS(self.doSuccessExecAutoRg) edpluginRg.connectFAILURE(self.doFailureExecAutoRg) edpluginRg.executeSynchronous() self.actualBestBuffer = self.dictRg.keys()[ self.dictRg.values().index(min(self.dictRg.values()))] else: self.actualBestBuffer = self.averBuffer else: self.synchronizePlugins() strError = "You should specify exactly 2 buffers for guessing best buffer, I got: " + ", ".join( self.buffers) self.WARNING(strError) self.lstProcessLog.append(strError) self.actualBestBuffer = self.averBuffer copy(self.actualBestBuffer, self.bestBuffer) self.lstProcessLog.append("Best buffer is %s" % self.actualBestBuffer) if self.isFailure() or not os.path.exists( self.bestBuffer) or not os.path.exists(self.sampleCurve): return edPluginDatop = self.loadPlugin(self.__strPluginDatop) edPluginDatop.dataInput = XSDataInputDatop( operation=XSDataString("SUB"), outputCurve=XSDataFile(XSDataString(self.subtractedCurve)), inputCurve=[ XSDataFile(XSDataString(self.sampleCurve)), XSDataFile(XSDataString(self.bestBuffer)) ]) edPluginDatop.connectSUCCESS(self.doSuccessExecDatop) edPluginDatop.connectFAILURE(self.doFailureExecDatop) edPluginDatop.executeSynchronous() if self.isFailure() or not os.path.exists(self.subtractedCurve): return self.headers = self.parseHeaders(self.sampleCurve) edpluginRg = self.loadPlugin(self.__strPluginAutoRg) edpluginRg.dataInput = XSDataInputAutoRg( inputCurve=[XSDataFile(XSDataString(self.subtractedCurve))]) edpluginRg.connectSUCCESS(self.doSuccessExecAutoRg) edpluginRg.connectFAILURE(self.doFailureExecAutoRg) edpluginRg.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(self.minimumCurveFileSize), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist" % dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist" % inp) time.sleep(1.0) shutil.copyfile(inp, dst) self.addExecutiveSummaryLine( "Got only one frame ... nothing to merge !!!") else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) self.datcmp(self.lstInput[0], oneFile) # edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile]) # edPluginExecAbsoluteFidelity.setDataInput(xsd) # edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") self.datcmp(self.lstInput[0], self.lstInput[1]) # edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ]) # edPluginExecAbsoluteFidelity.setDataInput(xsd) # edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) self.datcmp(self.lstInput[idx + 1], oneFile) # edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile]) # edPluginExecRelativeFidelity.setDataInput(xsd) # edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.addExecutiveSummaryLine("WARNING: " + self.strRadiationDamage) self.addExecutiveSummaryLine("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) # outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin( self.__strControlledPluginAutoSub) base = "_".join( os.path.basename( self.__class__.lastSample.path.value).split("_")[:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) self.xsdSubtractedCurve = XSDataFile(XSDataString(sub)) #self.curves.append(xsdSubtractedCurve) self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub( sampleCurve=self.__class__.lastSample, buffers=[ self.__class__.lastBuffer, self.dataInput.mergedCurve ], subtractedCurve=self.xsdSubtractedCurve) self.__edPluginExecAutoSub.connectSUCCESS( self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE( self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() if self.isFailure(): return self.__edPluginSaxsAnalysis = self.loadPlugin( self.__strControlledPluginSaxsAnalysis) self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis( scatterCurve=self.xsdSubtractedCurve, autoRg=self.autoRg, graphFormat=XSDataString("png")) self.__edPluginSaxsAnalysis.connectSUCCESS( self.doSuccessSaxsAnalysis) self.__edPluginSaxsAnalysis.connectFAILURE( self.doFailureSaxsAnalysis) self.__edPluginSaxsAnalysis.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve #self.__class__.lastSample = None #Information neededfor transfer to ISPyB self.forgetLastSample = True else: self.__class__.lastSample = self.dataInput.mergedCurve if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID: self.addExecutiveSummaryLine("Registering to ISPyB") self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged)) self.__class__.dictFrames[self.dataInput.mergedCurve] = { 'averaged': self.lstMerged, 'discarded': self.lstDiscarded } self.__edPluginSaxsISPyB = self.loadPlugin( self.__strControlledPluginSaxsISPyB) if len(self.lstInput) > 1: frameAverage = XSDataInteger(len(self.lstInput)) frameMerged = XSDataInteger(len(self.lstMerged)) else: frameMerged = frameAverage = XSDataInteger(1) self.curves = [XSDataFile(i.path) for i in self.lstInput] self.discardedCurves = [ XSDataFile(i.path) for i in self.lstDiscarded ] self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged] averageFilePath = None if self.strMergedFile is not None: averageFilePath = XSDataFile(XSDataString(self.strMergedFile)) self.sampleFrames = self.getAveragedFrameByFilename( self.__class__.lastSample) lastSample = None if self.__class__.lastSample is not None: lastSample = self.__class__.lastSample subtractedCurve = None if self.xsdSubtractedCurve is not None: subtractedCurve = self.xsdSubtractedCurve else: subtractedCurve = None xsdin = XSDataInputBioSaxsISPyBv1_0( sample=self.dataInput.sample, autoRg=self.autoRg, gnom=self.gnom, volume=self.volume, frameAverage=frameAverage, frameMerged=frameMerged, curves=self.curves, discardedFrames=self.discardedCurves, averagedFrames=self.mergedCurves, averageFilePath=averageFilePath, bufferFrames=self.bufferFrames, sampleFrames=self.sampleFrames, bestBuffer=self.xsBestBuffer, averageSample=lastSample, scatterPlot=self.xsScatterPlot, guinierPlot=self.xsGuinierPlot, kratkyPlot=self.xsKratkyPlot, densityPlot=self.xsDensityPlot, subtractedFilePath=subtractedCurve # destination=self.dataInput.sample.ispybDestination #duplicate, already in sample ) self.__edPluginSaxsISPyB.dataInput = xsdin self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB) self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB) self.__edPluginSaxsISPyB.execute() # transfer analysis data to correct location on nice if self.gnom is not None: self.outdir = os.path.join( os.path.dirname(os.path.dirname(self.lstStrInput[0])), "ednaAnalysis") basename = os.path.basename( os.path.splitext(self.gnom.gnomFile.path.value)[0]) self.outdir = os.path.join(self.outdir, basename) if not os.path.isdir(self.outdir): os.makedirs(self.outdir) #self.outFile = os.path.join(outdir, "NoResults.html") workingdir = os.path.dirname(self.gnom.gnomFile.path.value) self.pluginRsync = self.loadPlugin(self.cpRsync) self.pluginRsync.dataInput = XSDataInputRsync( source=XSDataFile(XSDataString(workingdir)), destination=XSDataFile(XSDataString(self.outdir)), options=XSDataString("-avx")) self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync) self.pluginRsync.connectFAILURE(self.doFailureExecRsync) self.pluginRsync.executeSynchronous() if self.forgetLastSample: #Also redefine dictionary to contain the buffer just processed? self.__class__.lastSample = None
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_0.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() # if self.isFailure(): # return if len(self.lstInput) == 1: shutil.copyfile(self.lstInput[0].path.value, self.dataInput.mergedCurve.path.value) else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for oneFile in self.lstInput[1:]: edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None) and (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstSummary.append("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_5.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(10000), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist" % dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist" % inp) time.sleep(1.0) shutil.copyfile(inp, dst) self.addExecutiveSummaryLine( "Got only one frame ... nothing to merge !!!") else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], oneFile]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") edPluginExecAbsoluteFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[0], self.lstInput[1]]) edPluginExecAbsoluteFidelity.setDataInput(xsd) edPluginExecAbsoluteFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecAbsoluteFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) edPluginExecRelativeFidelity = self.loadPlugin( self.__strControlledPluginDatcmp) xsd = XSDataInputDatcmp( inputCurve=[self.lstInput[idx + 1], oneFile]) edPluginExecRelativeFidelity.setDataInput(xsd) edPluginExecRelativeFidelity.connectFAILURE( self.doFailureExecDatcmp) edPluginExecRelativeFidelity.connectSUCCESS( self.doSuccessExecDatcmp) edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) else: break elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: break else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.addExecutiveSummaryLine("WARNING: " + self.strRadiationDamage) self.addExecutiveSummaryLine("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) # outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin( self.__strControlledPluginAutoSub) base = "_".join( os.path.basename( self.__class__.lastSample.path.value).split("_")[:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) xsdSubtractedCurve = XSDataFile(XSDataString(sub)) self.curves.append(xsdSubtractedCurve) self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub( sampleCurve=self.__class__.lastSample, buffers=[ self.__class__.lastBuffer, self.dataInput.mergedCurve ], subtractedCurve=xsdSubtractedCurve) self.__edPluginExecAutoSub.connectSUCCESS( self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE( self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() if self.isFailure(): return self.__edPluginSaxsAnalysis = self.loadPlugin( self.__strControlledPluginSaxsAnalysis) self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis( scatterCurve=xsdSubtractedCurve, autoRg=self.autoRg, graphFormat=XSDataString("png")) self.__edPluginSaxsAnalysis.connectSUCCESS( self.doSuccessSaxsAnalysis) self.__edPluginSaxsAnalysis.connectFAILURE( self.doFailureSaxsAnalysis) self.__edPluginSaxsAnalysis.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve self.__class__.lastSample = None else: self.__class__.lastSample = self.dataInput.mergedCurve if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID: self.addExecutiveSummaryLine("Registering to ISPyB") self.__edPluginSaxsISPyB = self.loadPlugin( self.__strControlledPluginSaxsISPyB) if len(self.lstInput) > 1: frameAverage = XSDataInteger(len(self.lstInput)) frameMerged = XSDataInteger(len(self.lstMerged)) else: frameMerged = frameAverage = XSDataInteger(1) self.curves = [XSDataFile(i.path) for i in self.lstInput] if self.strMergedFile is not None: self.curves.append(XSDataFile(XSDataString( self.strMergedFile))) xsdin = XSDataInputBioSaxsISPyBv1_0( sample=self.dataInput.sample, autoRg=self.autoRg, gnom=self.gnom, volume=self.volume, frameAverage=frameAverage, frameMerged=frameMerged, curves=self.curves, bestBuffer=self.xsBestBuffer, scatterPlot=self.xsScatterPlot, guinierPlot=self.xsGuinierPlot, kratkyPlot=self.xsKratkyPlot, densityPlot=self.xsDensityPlot # destination=self.dataInput.sample.ispybDestination #duplicate, already in sample ) self.__edPluginSaxsISPyB.dataInput = xsdin self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB) self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB) self.__edPluginSaxsISPyB.execute()