def testCheckParameters(self):
xsDataInput = XSDataInputDataver()
xsDataInput.inputCurve = [XSDataFile()]
xsDataInput.outputCurve = XSDataFile()
edPluginExecDataver = self.createPlugin()
edPluginExecDataver.setDataInput(xsDataInput)
edPluginExecDataver.checkParameters()
def testCheckParameters(self):
xsDataInput = XSDataInputDataver()
xsDataInput.inputCurve = [XSDataFile()]
xsDataInput.outputCurve = XSDataFile()
edPluginExecDataver = self.createPlugin()
edPluginExecDataver.setDataInput(xsDataInput)
edPluginExecDataver.checkParameters()
def average_buffers(self):
"""
Average out all buffers
"""
self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer))
xsdIn = XSDataInputDataver(inputCurve=[XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer])
if self.dataInput.bufferCurve:
xsdIn.outputCurve = self.dataInput.bufferCurve
else:
xsdIn.outputCurve = XSDataFile(XSDataString(self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer)))
self.__edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver)
self.__edPluginDatAver.dataInput = xsdIn
self.__edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
self.__edPluginDatAver.connectFAILURE(self.doFailureDatAver)
self.__edPluginDatAver.executeSynchronous()
def average_buffers(self):
"""
Average out all buffers
"""
self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer))
xsdIn = XSDataInputDataver(inputCurve=[XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer])
if self.dataInput.bufferCurve:
xsdIn.outputCurve = self.dataInput.bufferCurve
else:
xsdIn.outputCurve = XSDataFile(XSDataString(self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver_%04i.dat" % len(self.hplc_run.for_buffer)))
self.edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver)
self.edPluginDatAver.dataInput = xsdIn
self.edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
self.edPluginDatAver.connectFAILURE(self.doFailureDatAver)
self.edPluginDatAver.executeSynchronous()
def processRun(self, run):
run.dump_json()
hdf5 = run.save_hdf5()
self.json = os.path.splitext(hdf5)[0] + ".json"
self.xsDataResult.hplcFile = XSDataFile(XSDataString(hdf5))
self.xsDataResult.hplcImage = XSDataFile(XSDataString(run.make_plot()))
for group in run.analyse():
self.lstExecutiveSummary.append("Merging frames from %s to %s" %
(group[0], group[-1]))
xsdFrames = [
XSDataFile(XSDataString(run.frames[i].subtracted))
for i in group
]
outname = os.path.splitext(run.frames[
group[0]].subtracted)[0] + "_aver_%s.dat" % group[-1]
edpugin = self.loadPlugin(self.strControlledPluginDatAver)
edpugin.dataInput = XSDataInputDataver(outputCurve=XSDataFile(
XSDataString(outname)),
inputCurve=xsdFrames)
edpugin.connectSUCCESS(self.doSuccessDatAver)
edpugin.connectFAILURE(self.doFailureDatAver)
edpugin.execute()
run.merge_curves.append(outname)
run.merge_analysis[outname] = None
run.merge_Rg[outname] = None
run.merge_framesDIC[outname] = [group[0], group[-1]]
# Append to hdf5
run.append_hdf5()
def processRun(self, run):
# EDPluginBioSaxsHPLCv1_5.dictHPLC[self.runId].reset()
for idx in run.frames:
run.frames[idx].purge_memory()
run.dump_json()
hdf5 = run.save_hdf5(name=self.__class__.__name__)
self.json = os.path.splitext(hdf5)[0] + ".json"
self.xsDataResult.hplcFile = XSDataFile(XSDataString(hdf5))
self.xsDataResult.hplcImage = XSDataFile(XSDataString(run.make_plot()))
peaks = []
try:
peaks = run.analyse()
except Exception as error:
traceback.print_stack()
self.ERROR("EDPluginBioSaxsFlushHPLCv1_5: Error in run.analyse() %s" % error)
for group in peaks:
self.lstExecutiveSummary.append("Merging frames from %s to %s" % (group[0], group[-1]))
xsdFrames = [XSDataFile(XSDataString(run.frames[i].subtracted)) for i in group]
outname = os.path.splitext(run.frames[group[0]].subtracted)[0] + "_aver_%s.dat" % group[-1]
edpugin = self.loadPlugin(self.strControlledPluginDatAver)
edpugin.dataInput = XSDataInputDataver(outputCurve=XSDataFile(XSDataString(outname)), inputCurve=xsdFrames)
edpugin.connectSUCCESS(self.doSuccessDatAver)
edpugin.connectFAILURE(self.doFailureDatAver)
edpugin.execute()
run.merge_curves.append(outname)
run.merge_analysis[outname] = None
run.merge_Rg[outname] = None
run.merge_framesDIC[outname] = [group[0], group[-1]]
# Append to hdf5
run.append_hdf5()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_3.process")
xsdwf = XSDataInputWaitMultiFile(timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist"%dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist"%inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None):
if self.absoluteFidelity is not None :
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None) :
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None) :
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.lstSummary.append("WARNING: " + self.strRadiationDamage)
self.lstSummary.append("Merging files: " + " ".join([os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
#outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(self.__strControlledPluginAutoSub)
base = "_".join(os.path.basename(self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff)
xsd = XSDataInputAutoSub(sampleCurve=self.__class__.lastSample,
buffers=[self.__class__.lastBuffer, self.dataInput.mergedCurve],
subtractedCurve=XSDataFile(XSDataString(sub))
)
self.__edPluginExecAutoSub.setDataInput(xsd)
self.__edPluginExecAutoSub.connectSUCCESS(self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
self.__class__.lastSample = None
else:
self.__class__.lastSample = self.dataInput.mergedCurve
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_1.process")
xsdwf = XSDataInputWaitMultiFile(timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
shutil.copyfile(self.lstInput[0].path.value, self.dataInput.mergedCurve.path.value)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None):
if self.absoluteFidelity is not None :
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None) :
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None) :
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities]))
if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.lstSummary.append("WARNING: " + self.strRadiationDamage)
self.lstSummary.append("Merging files: " + " ".join([os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
shutil.copyfile(self.lstMerged[0].path.value, self.dataInput.mergedCurve.path.value)
else:
self.__edPluginExecDataver = self.loadPlugin(self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
#outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if self.isFailure():
return
self.__edPluginExecAutoRg = self.loadPlugin(self.__strControlledPluginAutoRG)
xsd = XSDataInputAutoRg(inputCurve=[self.dataInput.mergedCurve])
self.__edPluginExecAutoRg.setDataInput(xsd)
self.__edPluginExecAutoRg.connectSUCCESS(self.doSuccessExecAutoRg)
self.__edPluginExecAutoRg.connectFAILURE(self.doFailureExecAutoRg)
self.__edPluginExecAutoRg.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginAutoSubv1_0.process")
basename = os.path.splitext(os.path.basename(self.sampleCurve))[0]
self.bestBuffer = os.path.join(self.outdir,
"%s_bestbuffer.dat" % basename)
self.averBuffer = os.path.join(self.outdir,
"%s_averbuffer.dat" % basename)
for idx, name in enumerate(self.buffers):
copy(name,
os.path.join(self.outdir, "%s_buf%i.dat" % (basename, idx)))
if len(self.buffers) == 1:
self.actualBestBuffer = self.buffers[0]
else:
self.__edPluginDataver.dataInput = XSDataInputDataver(
inputCurve=self.dataInput.buffers,
outputCurve=XSDataFile(XSDataString(self.averBuffer)))
self.__edPluginDataver.connectSUCCESS(self.doSuccessExecDataver)
self.__edPluginDataver.connectFAILURE(self.doFailureExecDataver)
self.__edPluginDataver.execute()
if len(self.buffers) == 2:
edPluginDatcmp = self.loadPlugin(self.__strPluginDatcmp)
edPluginDatcmp.dataInput = XSDataInputDatcmp(
inputCurve=self.dataInput.buffers)
edPluginDatcmp.connectSUCCESS(self.doSuccessExecDatcmp)
edPluginDatcmp.connectFAILURE(self.doFailureExecDatcmp)
edPluginDatcmp.executeSynchronous()
if self.isFailure() or (self.fidelity is None):
return
if self.fidelity < self.BUFFER_SIMILARITY: #buffer are not the same: keeping the one with lowest Rg/I0
edpluginRg = self.loadPlugin(self.__strPluginAutoRg)
edpluginRg.dataInput = XSDataInputAutoRg(
inputCurve=self.dataInput.buffers)
edpluginRg.connectSUCCESS(self.doSuccessExecAutoRg)
edpluginRg.connectFAILURE(self.doFailureExecAutoRg)
edpluginRg.executeSynchronous()
self.actualBestBuffer = self.dictRg.keys()[
self.dictRg.values().index(min(self.dictRg.values()))]
else:
self.actualBestBuffer = self.averBuffer
else:
self.synchronizePlugins()
strError = "You should specify exactly 2 buffers for guessing best buffer, I got: " + ", ".join(
self.buffers)
self.WARNING(strError)
self.lstProcessLog.append(strError)
self.actualBestBuffer = self.averBuffer
copy(self.actualBestBuffer, self.bestBuffer)
self.lstProcessLog.append("Best buffer is %s" % self.actualBestBuffer)
if self.isFailure() or not os.path.exists(
self.bestBuffer) or not os.path.exists(self.sampleCurve):
return
edPluginDatop = self.loadPlugin(self.__strPluginDatop)
edPluginDatop.dataInput = XSDataInputDatop(
operation=XSDataString("SUB"),
outputCurve=XSDataFile(XSDataString(self.subtractedCurve)),
inputCurve=[
XSDataFile(XSDataString(self.sampleCurve)),
XSDataFile(XSDataString(self.bestBuffer))
])
edPluginDatop.connectSUCCESS(self.doSuccessExecDatop)
edPluginDatop.connectFAILURE(self.doFailureExecDatop)
edPluginDatop.executeSynchronous()
if self.isFailure() or not os.path.exists(self.subtractedCurve):
return
self.headers = self.parseHeaders(self.sampleCurve)
edpluginRg = self.loadPlugin(self.__strPluginAutoRg)
edpluginRg.dataInput = XSDataInputAutoRg(
inputCurve=[XSDataFile(XSDataString(self.subtractedCurve))])
edpluginRg.connectSUCCESS(self.doSuccessExecAutoRg)
edpluginRg.connectFAILURE(self.doFailureExecAutoRg)
edpluginRg.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(self.minimumCurveFileSize),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
self.addExecutiveSummaryLine(
"Got only one frame ... nothing to merge !!!")
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
self.datcmp(self.lstInput[0], oneFile)
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
self.datcmp(self.lstInput[0], self.lstInput[1])
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
self.datcmp(self.lstInput[idx + 1], oneFile)
# edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
# edPluginExecRelativeFidelity.setDataInput(xsd)
# edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.addExecutiveSummaryLine("WARNING: " +
self.strRadiationDamage)
self.addExecutiveSummaryLine("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
# outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
self.xsdSubtractedCurve = XSDataFile(XSDataString(sub))
#self.curves.append(xsdSubtractedCurve)
self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer, self.dataInput.mergedCurve
],
subtractedCurve=self.xsdSubtractedCurve)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
if self.isFailure():
return
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.__strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
scatterCurve=self.xsdSubtractedCurve,
autoRg=self.autoRg,
graphFormat=XSDataString("png"))
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
#self.__class__.lastSample = None #Information neededfor transfer to ISPyB
self.forgetLastSample = True
else:
self.__class__.lastSample = self.dataInput.mergedCurve
if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
self.addExecutiveSummaryLine("Registering to ISPyB")
self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged))
self.__class__.dictFrames[self.dataInput.mergedCurve] = {
'averaged': self.lstMerged,
'discarded': self.lstDiscarded
}
self.__edPluginSaxsISPyB = self.loadPlugin(
self.__strControlledPluginSaxsISPyB)
if len(self.lstInput) > 1:
frameAverage = XSDataInteger(len(self.lstInput))
frameMerged = XSDataInteger(len(self.lstMerged))
else:
frameMerged = frameAverage = XSDataInteger(1)
self.curves = [XSDataFile(i.path) for i in self.lstInput]
self.discardedCurves = [
XSDataFile(i.path) for i in self.lstDiscarded
]
self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged]
averageFilePath = None
if self.strMergedFile is not None:
averageFilePath = XSDataFile(XSDataString(self.strMergedFile))
self.sampleFrames = self.getAveragedFrameByFilename(
self.__class__.lastSample)
lastSample = None
if self.__class__.lastSample is not None:
lastSample = self.__class__.lastSample
subtractedCurve = None
if self.xsdSubtractedCurve is not None:
subtractedCurve = self.xsdSubtractedCurve
else:
subtractedCurve = None
xsdin = XSDataInputBioSaxsISPyBv1_0(
sample=self.dataInput.sample,
autoRg=self.autoRg,
gnom=self.gnom,
volume=self.volume,
frameAverage=frameAverage,
frameMerged=frameMerged,
curves=self.curves,
discardedFrames=self.discardedCurves,
averagedFrames=self.mergedCurves,
averageFilePath=averageFilePath,
bufferFrames=self.bufferFrames,
sampleFrames=self.sampleFrames,
bestBuffer=self.xsBestBuffer,
averageSample=lastSample,
scatterPlot=self.xsScatterPlot,
guinierPlot=self.xsGuinierPlot,
kratkyPlot=self.xsKratkyPlot,
densityPlot=self.xsDensityPlot,
subtractedFilePath=subtractedCurve
# destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
)
self.__edPluginSaxsISPyB.dataInput = xsdin
self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
self.__edPluginSaxsISPyB.execute()
# transfer analysis data to correct location on nice
if self.gnom is not None:
self.outdir = os.path.join(
os.path.dirname(os.path.dirname(self.lstStrInput[0])),
"ednaAnalysis")
basename = os.path.basename(
os.path.splitext(self.gnom.gnomFile.path.value)[0])
self.outdir = os.path.join(self.outdir, basename)
if not os.path.isdir(self.outdir):
os.makedirs(self.outdir)
#self.outFile = os.path.join(outdir, "NoResults.html")
workingdir = os.path.dirname(self.gnom.gnomFile.path.value)
self.pluginRsync = self.loadPlugin(self.cpRsync)
self.pluginRsync.dataInput = XSDataInputRsync(
source=XSDataFile(XSDataString(workingdir)),
destination=XSDataFile(XSDataString(self.outdir)),
options=XSDataString("-avx"))
self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync)
self.pluginRsync.connectFAILURE(self.doFailureExecRsync)
self.pluginRsync.executeSynchronous()
if self.forgetLastSample:
#Also redefine dictionary to contain the buffer just processed?
self.__class__.lastSample = None
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_0.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
# if self.isFailure():
# return
if len(self.lstInput) == 1:
shutil.copyfile(self.lstInput[0].path.value,
self.dataInput.mergedCurve.path.value)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for oneFile in self.lstInput[1:]:
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity
is not None) and (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstSummary.append("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_5.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
self.addExecutiveSummaryLine(
"Got only one frame ... nothing to merge !!!")
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], self.lstInput[1]])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.addExecutiveSummaryLine("WARNING: " +
self.strRadiationDamage)
self.addExecutiveSummaryLine("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
# outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
xsdSubtractedCurve = XSDataFile(XSDataString(sub))
self.curves.append(xsdSubtractedCurve)
self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer, self.dataInput.mergedCurve
],
subtractedCurve=xsdSubtractedCurve)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
if self.isFailure():
return
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.__strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
scatterCurve=xsdSubtractedCurve,
autoRg=self.autoRg,
graphFormat=XSDataString("png"))
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
self.__class__.lastSample = None
else:
self.__class__.lastSample = self.dataInput.mergedCurve
if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
self.addExecutiveSummaryLine("Registering to ISPyB")
self.__edPluginSaxsISPyB = self.loadPlugin(
self.__strControlledPluginSaxsISPyB)
if len(self.lstInput) > 1:
frameAverage = XSDataInteger(len(self.lstInput))
frameMerged = XSDataInteger(len(self.lstMerged))
else:
frameMerged = frameAverage = XSDataInteger(1)
self.curves = [XSDataFile(i.path) for i in self.lstInput]
if self.strMergedFile is not None:
self.curves.append(XSDataFile(XSDataString(
self.strMergedFile)))
xsdin = XSDataInputBioSaxsISPyBv1_0(
sample=self.dataInput.sample,
autoRg=self.autoRg,
gnom=self.gnom,
volume=self.volume,
frameAverage=frameAverage,
frameMerged=frameMerged,
curves=self.curves,
bestBuffer=self.xsBestBuffer,
scatterPlot=self.xsScatterPlot,
guinierPlot=self.xsGuinierPlot,
kratkyPlot=self.xsKratkyPlot,
densityPlot=self.xsDensityPlot
# destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
)
self.__edPluginSaxsISPyB.dataInput = xsdin
self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
self.__edPluginSaxsISPyB.execute()
