def functError(self, xmlin):
try:
xsd = XSDataInputBioSaxsProcessOneFilev1_0.parseString(xmlin)
filename = xsd.rawImage.path.value
except Exception:
filename = xmlin
self.ERROR("Processing failed for: %s" % filename)
def functError(self, xmlin):
try:
xsd = XSDataInputBioSaxsProcessOneFilev1_0.parseString(xmlin)
filename = xsd.rawImage.path.value
except Exception:
filename = xmlin
self.ERROR("Processing failed for: %s" % filename)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsReduceFileSeriev1_0.process")
if self.bSkipProcess:
return
for oneRaw, oneCor, oneInt, oneCrv, oneLog in zip(self.lstRawImg, self.lstCorImg, self.lstIntImg, self.lstIntCrv, self.lstLogFil):
pluginProcessOneFile = self.loadPlugin(self.__strControlledPluginProcessOneFile)
xsd = XSDataInputBioSaxsProcessOneFilev1_0(integratedCurve=oneCrv,
integratedImage=oneInt,
normalizedImage=oneCor,
rawImageSize=self.rawImageSize,
experimentSetup=self.experimentSetup,
sample=self.sample,
rawImage=oneRaw,
logFile=oneLog)
pluginProcessOneFile.dataInput = xsd
pluginProcessOneFile.connectSUCCESS(self.doSuccessExecProcessOneFile)
pluginProcessOneFile.connectFAILURE(self.doFailureExecProcessOneFile)
pluginProcessOneFile.execute()
self.synchronizePlugins()
self.lstCurves.sort()
xsdMerge = XSDataInputBioSaxsSmartMergev1_0()
xsdMerge.absoluteFidelity = self.dataInput.absoluteFidelity
xsdMerge.relativeFidelity = self.dataInput.relativeFidelity
xsdMerge.inputCurves = [XSDataFile(XSDataString(i)) for i in self.lstCurves]
xsdMerge.mergedCurve = XSDataFile(XSDataString(self.strMergedCurve))
edPluginExecSmartMerge = self.loadPlugin(self.__strControlledPluginSmartMerge)
edPluginExecSmartMerge.dataInput = xsdMerge
edPluginExecSmartMerge.connectSUCCESS(self.doSuccessExecSmartMerge)
edPluginExecSmartMerge.connectFAILURE(self.doFailureExecSmartMerge)
edPluginExecSmartMerge.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_3.process")
xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(rawImage=self.dataInput.rawImage,
sample=self.dataInput.sample,
experimentSetup=self.dataInput.experimentSetup,
rawImageSize=self.dataInput.rawImageSize,
normalizedImage=self.dataInput.normalizedImage,
integratedCurve=self.dataInput.integratedCurve,
runId=self.dataInput.runId,
frameId=self.dataInput.frameId)
self.edPluginProcessOneFile = self.loadPlugin(self.strControlledPluginProcessOneFile)
self.edPluginProcessOneFile.dataInput = xsdIn
self.edPluginProcessOneFile.connectSUCCESS(self.doSuccessProcessOneFile)
self.edPluginProcessOneFile.connectFAILURE(self.doFailureProcessOneFile)
self.edPluginProcessOneFile.executeSynchronous()
if self.isFailure():
return
xsdIn = XSDataInputDatcmp(inputCurve=[XSDataFile(XSDataString(self.hplc_run.first_curve)),
XSDataFile(XSDataString(self.curve))])
self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
self.edPluginDatCmp.dataInput = xsdIn
self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
self.edPluginDatCmp.executeSynchronous()
if self.isFailure() or self.isBuffer:
return
if self.dataInput.subtractedCurve is not None:
subtracted = self.dataInput.subtractedCurve.path.value
else:
subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
if self.hplc_run.buffer is not None:
xsdIn = XSDataInputDatop(inputCurve=[XSDataFile(XSDataString(self.curve)),
XSDataFile(XSDataString(self.hplc_run.buffer))],
outputCurve=XSDataFile(XSDataString(subtracted)),
operation=XSDataString("sub"))
else:
xsdIn = XSDataInputDatop(inputCurve=[XSDataFile(XSDataString(self.curve)),
XSDataFile(XSDataString(self.hplc_run.first_curve))],
outputCurve=XSDataFile(XSDataString(subtracted)),
operation=XSDataString("sub"))
self.edPluginDatop = self.loadPlugin(self.strControlledPluginDatop)
self.edPluginDatop.dataInput = xsdIn
self.edPluginDatop.connectSUCCESS(self.doSuccessDatop)
self.edPluginDatop.connectFAILURE(self.doFailureDatop)
self.edPluginDatop.executeSynchronous()
if self.subtracted and os.path.exists(self.subtracted):
self.edPluginAutoRg = self.loadPlugin(self.strControlledPluginAutoRg)
self.edPluginAutoRg.dataInput = XSDataInputAutoRg(inputCurve=[XSDataFile(XSDataString(self.subtracted))])
self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg)
self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg)
self.edPluginAutoRg.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_4.process")
xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(rawImage=self.dataInput.rawImage,
sample=self.dataInput.sample,
experimentSetup=self.dataInput.experimentSetup,
rawImageSize=self.dataInput.rawImageSize,
normalizedImage=self.dataInput.normalizedImage,
integratedCurve=self.dataInput.integratedCurve,
runId=self.dataInput.runId,
frameId=self.dataInput.frameId)
self.edPluginProcessOneFile = self.loadPlugin(self.strControlledPluginProcessOneFile)
self.edPluginProcessOneFile.dataInput = xsdIn
self.edPluginProcessOneFile.connectSUCCESS(self.doSuccessProcessOneFile)
self.edPluginProcessOneFile.connectFAILURE(self.doFailureProcessOneFile)
self.edPluginProcessOneFile.executeSynchronous()
if self.isFailure():
return
xsdIn = XSDataInputDatcmp(inputCurve=[XSDataFile(XSDataString(self.hplc_run.first_curve)),
XSDataFile(XSDataString(self.curve))])
self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
self.edPluginDatCmp.dataInput = xsdIn
self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
self.edPluginDatCmp.executeSynchronous()
if self.isFailure() or self.isBuffer:
return
#######################
# # DatOp subtraction ##
#######################
if self.dataInput.subtractedCurve is not None:
self.subtracted = self.dataInput.subtractedCurve.path.value
else:
self.subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
Isub = self.intensity - self.hplc_run.buffer_I
StdErr = numpy.sqrt(self.stdError * self.stdError + \
self.hplc_run.buffer_Stdev * self.hplc_run.buffer_Stdev)
with open(self.subtracted, "w") as outfile:
numpy.savetxt(outfile, numpy.vstack((self.hplc_run.q, Isub, StdErr)).T)
self.xsDataResult.subtractedCurve = XSDataFile(XSDataString(self.subtracted))
self.frame.subtracted = self.subtracted
if self.subtracted and os.path.exists(self.subtracted):
self.edPluginAutoRg = self.loadPlugin(self.strControlledPluginAutoRg)
self.edPluginAutoRg.dataInput = XSDataInputAutoRg(inputCurve=[XSDataFile(XSDataString(self.subtracted))])
self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg)
self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg)
self.edPluginAutoRg.executeSynchronous()
def testCheckParameters(self):
xsDataInput = XSDataInputBioSaxsProcessOneFilev1_0()
xsDataInput.rawImage = XSDataImage()
xsDataInput.sample = XSDataBioSaxsSample()
xsDataInput.experimentSetup = XSDataBioSaxsExperimentSetup()
xsDataInput.logFile = XSDataFile()
xsDataInput.normalizedImage = XSDataImage()
xsDataInput.rawImageSize = XSDataInteger()
# xsDataInput.beamStopDiode = XSDataDouble()
# xsDataInput.normalizationFactor = XSDataDouble()
# xsDataInput.machineCurrent = XSDataDouble()
# xsDataInput.maskFile = XSDataImage()
# xsDataInput.detectorDistance = XSDataLength()
# xsDataInput.wavelength = XSDataWavelength()
# xsDataInput.pixelSize_1 = XSDataLength()
# xsDataInput.pixelSize_2 = XSDataLength()
# xsDataInput.beamCenter_1 = XSDataDouble()
# xsDataInput.beamCenter_2 = XSDataDouble()
edPluginExecBioSaxsNormalize = self.createPlugin()
edPluginExecBioSaxsNormalize.setDataInput(xsDataInput)
edPluginExecBioSaxsNormalize.checkParameters()
delay = float(arg.split("=", 1)[1])
elif os.path.isfile(arg):
filenames.append(os.path.abspath(arg))
edna = PyTango.DeviceProxy(device)
edna.initPlugin(plugin)
t0 = time.time()
totaltime = 0.0
listjobs = []
if filenames:
reprocessDirname = "reprocess-%s" % time.strftime("%Y%m%d-%Hh%M", time.localtime())
for fn in filenames:
if fn.endswith(".xml"):
dirname, xmlbase = os.path.split(fn)
xsd = XSDataInputBioSaxsProcessOneFilev1_0.parseFile(fn)
if xsd.experimentSetup is None:
print("XML file %s is not a good XSD file")
continue
xsd.experimentSetup.maskFile.path.value = mask
oldRawdir, rawfile = os.path.split(xsd.rawImage.path.value)
newraw = os.path.join(dirname, rawfile)
if not os.path.isfile(newraw):
print("new raw %s does dot exist: skipping !!!" % newraw)
continue
xsd.rawImage.path.value = newraw
oldbase = os.path.dirname(oldRawdir)
newbase = os.path.join(os.path.dirname(dirname), reprocessDirname)
xmldir = os.path.join(newbase, "raw")
if not os.path.isdir(newbase):
os.makedirs(xmldir)
def toXml(self, filename):
"""Here we create the XML string to be passed to the EDNA plugin from the input filename
This can / should be modified by the final user
@param filename: full path of the input file
@type filename: python string representing the path
@rtype: XML string
@return: python string
"""
# Is the file to process?
dirname, basename = os.path.split(filename)
rootDir, dirname = os.path.split(dirname)
basename, ext = os.path.splitext(basename)
if not ((dirname.lower() == "raw") and (ext.lower() == ".edf")):
self.DEBUG("Not processing %s" % filename)
return
existing2DImage = os .path.join(rootDir, "2d", basename + ".edf")
if os.path.isfile(existing2DImage):
# print existing2DImage
title = None
for line in open(existing2DImage):
if len(line) > 512:
break
else:
line = line.strip()
if line.startswith("title"):
title = line.split("=", 1)[-1]
if line.startswith("Center_1")and (self.beamCenter_1 is None):
try:
self.experimentSetup.beamCenter_1 = XSDataDouble(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
elif line.startswith("Center_2")and (self.beamCenter_2 is None):
try:
self.experimentSetup.beamCenter_2 = XSDataDouble(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
elif line.startswith("PSize_1")and (self.pixelSize_1 is None):
try:
self.experimentSetup.pixelSize_1 = XSDataLength(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
elif line.startswith("PSize_2") and (self.pixelSize_2 is None):
try:
self.experimentSetup.pixelSize_2 = XSDataLength(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
elif line.startswith("SampleDistance") and (self.detectorDistance is None) :
try:
self.experimentSetup.detectorDistance = XSDataLength(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
elif line.startswith("WaveLength")and (self.wavelength is None):
try:
self.experimentSetup.wavelength = XSDataWavelength(float(line.split("=", 1)[-1].split()[0]))
except Exception:
self.ERROR("Reading from header: not a float " + line)
if title:
for block in title.split(","):
if "=" in block:
key, value = block.split("=", 1)
elif ":" in block:
key, value = block.split(":", 1)
else:
key = None
if key:
key = key.strip()
value = value.strip()
# print key, value
if key == "DiodeCurr":
try:
self.experimentSetup.beamStopDiode = XSDataDouble(float(value))
except Exception:
self.ERROR("Unable to convert %s to float" % block)
elif key == "MachCurr":
try:
self.experimentSetup.machineCurrent = XSDataDouble(float(value.split()[0]))
except Exception:
self.ERROR("Unable to convert %s to float" % block)
elif key == "Concentration":
try:
self.sample.concentration = XSDataDouble(float(value.split()[0]))
except Exception:
self.ERROR("Unable to convert %s to float" % block)
elif (key == "Comments") and (self.comment is None):
self.sample.comments = XSDataString(value.strip())
elif (key == "Code") and (self.code is None) :
self.sample.code = XSDataString(value.strip())
elif (key == "Mask") and (self.maskFile is None):
self.experimentSetup.maskFile = XSDataImage(XSDataString(value.strip()))
elif (key == "Normalisation") and (self.normalization is None):
try:
self.experimentSetup.normalizationFactor = XSDataDouble(float(value.split()[0]))
except Exception:
self.ERROR("Unable to convert %s to float" % block)
xsd = XSDataInputBioSaxsProcessOneFilev1_0()
xsd.rawImage = XSDataImage(XSDataString(filename))
xsd.rawImageSize = XSDataInteger(self.imageSize)
xsd.experimentSetup = self.experimentSetup
xsd.sample = self.sample
dir1d = os.path.join(rootDir, self.dest1D)
dir2d = os.path.join(rootDir, self.dest2D)
dirMisc = os.path.join(rootDir, self.destMisc)
if not os.path.exists(dir1d):
os.makedirs(dir1d)
if not os.path.exists(dir2d):
os.makedirs(dir2d)
if not os.path.exists(dirMisc):
os.makedirs(dirMisc)
xsd.normalizedImage = XSDataImage(XSDataString(os.path.join(dir2d, basename + ".edf")))
xsd.integratedImage = XSDataImage(XSDataString(os.path.join(dirMisc, basename + ".ang")))
xsd.integratedCurve = XSDataFile(XSDataString(os.path.join(dir1d, basename + ".dat")))
# print xsd.marshal()
return xsd.marshal()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsHPLCv1_5.process")
xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(rawImage=self.dataInput.rawImage,
sample=self.dataInput.sample,
experimentSetup=self.dataInput.experimentSetup,
rawImageSize=self.dataInput.rawImageSize,
normalizedImage=self.dataInput.normalizedImage,
integratedCurve=self.dataInput.integratedCurve,
runId=self.dataInput.runId,
frameId=self.dataInput.frameId)
self.edPluginProcessOneFile = self.loadPlugin(self.strControlledPluginProcessOneFile)
self.edPluginProcessOneFile.dataInput = xsdIn
self.edPluginProcessOneFile.connectSUCCESS(self.doSuccessProcessOneFile)
self.edPluginProcessOneFile.connectFAILURE(self.doFailureProcessOneFile)
self.edPluginProcessOneFile.executeSynchronous()
if self.isFailure():
return
if self.hplc_run.first_curve == self.curve:
return
# xsdIn = XSDataInputDatcmp(inputCurve=[XSDataFile(XSDataString(self.hplc_run.first_curve)),
# XSDataFile(XSDataString(self.curve))])
# self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
# self.edPluginDatCmp.dataInput = xsdIn
# self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
# self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
# self.edPluginDatCmp.executeSynchronous()
self.datcmp(self.hplc_run.firstCurveIntensity, self.intensity)
if self.isFailure() or self.isBuffer:
return
#######################
# # DatOp subtraction ##
#######################
if self.dataInput.subtractedCurve is not None:
self.subtracted = self.dataInput.subtractedCurve.path.value
else:
self.subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
Isub = self.intensity - self.hplc_run.buffer_I
StdErr = numpy.sqrt(self.stdError * self.stdError + \
self.hplc_run.buffer_Stdev * self.hplc_run.buffer_Stdev)
self.SAXSdata= numpy.vstack((self.hplc_run.q, Isub, StdErr)).T
with open(self.subtracted, "w") as outfile:
#print "Writing file"
numpy.savetxt(outfile, self.SAXSdata)
self.xsDataResult.subtractedCurve = XSDataFile(XSDataString(self.subtracted))
self.frame.subtracted = self.subtracted
if self.subtracted and os.path.exists(self.subtracted) and self.ATSASRg:
self.edPluginAutoRg = self.loadPlugin(self.strControlledPluginAutoRg)
self.edPluginAutoRg.dataInput = XSDataInputAutoRg(inputCurve=[XSDataFile(XSDataString(self.subtracted))])
self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg)
self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg)
self.edPluginAutoRg.executeSynchronous()
try:
#print self.subtracted, autoRg(self.SAXSdata)
rg_result = autoRg(self.SAXSdata)
self.frame.RgF, self.frame.Rg_StdevF, self.frame.I0F, self.frame.I0_StdevF, self.frame.Rg_imin, self.frame.Rg_imax = rg_result[:6]
except InsufficientDataError:
print "Not enough usable data to run autorg"
except SystemError as SE:
print "System Error"
print SE
except Exception as Err:
print Err
raise
else:
if self.ATSASRg == False:
self.frame.Rg = max(0,self.frame.RgF)
self.frame.Rg_Stdev = max(0,self.frame.Rg_StdevF)
self.frame.I0 = max(0,self.frame.I0F)
self.frame.I0_Stdev = max(0,self.frame.I0_StdevF)
self.frame.quality = 0
"""
Calculate the invariants Vc and Qr from the Rambo&Tainer 2013 Paper,
also the the mass estimate based on Qr for proteins
"""
if self.subtracted and os.path.exists(self.subtracted):
self.subtracted_data = numpy.loadtxt(self.subtracted)
if self.subtracted_data is not None and\
self.frame.RgF and self.frame.Rg_StdevF and self.frame.I0F and self.frame.I0_StdevF:
dictRTI = RamboTainerInvariant(self.subtracted_data, self.frame.RgF,
self.frame.Rg_StdevF, self.frame.I0F,
self.frame.I0_StdevF, self.frame.Rg_imin)
# {'Vc': vc[0], 'dVc': vc[1], 'Qr': qr, 'dQr': dqr, 'mass': mass, 'dmass': dmass}
self.frame.Vc = dictRTI.get("Vc")
self.frame.Vc_Stdev = dictRTI.get("dVc")
self.frame.Qr = dictRTI.get("Qr")
self.frame.Qr_Stdev = dictRTI.get("dQ")
self.frame.mass = dictRTI.get("mass")
self.frame.mass_Stdev = dictRTI.get("dmass")
xsdRTI = XSDataRamboTainer(vc=XSDataDouble(self.frame.Vc),
qr=XSDataDouble(self.frame.Qr),
mass=XSDataDouble(self.frame.mass),
dvc=XSDataDouble(self.frame.Vc_Stdev),
dqr=XSDataDouble(self.frame.Qr_Stdev),
dmass=XSDataDouble(self.frame.mass_Stdev))
self.xsDataResult.rti = xsdRTI