def read_next(self,progressReporter=None):
df = None
with file(self.path,'r') as stream:
reader = _odm.getODMDataReader(stream,chunksize=1000, skipDataRows=self.nlinesRead)
chunks = [chunk for chunk in reader if chunk is not None]
if len(chunks) > 0:
df = _pd.concat(chunks)
df = df[df.intensityProfile.map(len) > 0]
if (df is not None):
self.nlinesRead += len(df.index)
if (progressReporter):
progressReporter.message("%i new lines read" % len(df.index))
return df
def read(self,path):
super(CsvReader, self).read(path)
self._setStatusMessage("reading...")
reader = odm.getODMDataReader(path,chunksize=500)
lineCount = float(sum(1 for line in open(path)))
chunks = []
linesRead = 1
for chunk in reader:
linesRead += 500
self.appendChunkToData(chunk)
self._setStatusMessage("%i lines read" % linesRead)
self._setProgress(linesRead/lineCount * 100)
self._setStatusMessage("File loaded")
self._setProgress(100)
def fitRawODMData(filename,settingsFile=None,fitSettingsFile=None,referenceIPDataFile=None):
"""
This script opens and analyzes the target data.csv file produced by LabVIEW and
analyzes the optical displacement of a peak relative to another peak.
Results are saved to the same folder as the original data file.
Parameters
----------
filename: string
Path to the data.csv file produced by LabVIEW that contains the raw
Optical Displacement Measurement Data.
referenceIPDataFile: string
Path the data.csv file of which the first intensity profile will be used
as a reference for initializing the fit function.
Returns
-------
dataframe: pandas.DataFrame
A dataframe that contains the raw data, the calculated displacements,
curve fit results and other diagnostic data.
movingPeakFitSettings : CurveFitSettings instance
The curve fit settings used for the moving peak.
referencePeakFitSettings: CurveFitSettings instance
The curve fit settings used for the reference peak.
measurementName: string
The name of the measurement (deduced from the folder name of the csv file)
"""
commonPath = os.path.abspath(os.path.split(filename)[0])
measurementName = os.path.split(os.path.split(filename)[0])[1]
globalSettings = odm.CurveFitSettings.loadFromFileOrCreateDefault('./CurveFitScriptSettings.ini')
if (settingsFile is not None):
print "reading settings from %s" % settingsFile
settings = odm.CurveFitSettings.loadFromFile(settingsFile)
else:
settings = odm.CurveFitSettings.loadFromFileOrCreateDefault(commonPath + '/odmSettings.ini',prototype=globalSettings)
gui.getSettingsFromUser(settings)
df = odm.readODMData(filename)
if referenceIPDataFile is not None:
print "using the first profile from %s for initializing the fit functions" % referenceIPDataFile
for chunk in odm.getODMDataReader(referenceIPDataFile, chunksize=1):
referenceIntensityProfile = chunk.intensityProfile.iloc[0]
break
else:
referenceIntensityProfile = df.intensityProfile.iloc[0]
if (fitSettingsFile is not None):
with file(fitSettingsFile,'r') as f:
print "reading fit settings from %s" % fitSettingsFile
settingsDict = pickle.load(f)
movingPeakFitSettings = settingsDict['movingPeakFitSettings']
referencePeakFitSettings = settingsDict['referencePeakFitSettings']
else:
movingPeakFitFunction = ff.createFitFunction(settings.defaultFitFunction)
movingPeakFitSettings = gui.getPeakFitSettingsFromUser(referenceIntensityProfile,movingPeakFitFunction,
estimatorPromptPrefix="Moving peak:",
windowTitle="Moving Peak estimates")
try:
referencePeakFitFunction = ff.createFitFunction(settings.defaultFitFunction)
referencePeakFitSettings = gui.getPeakFitSettingsFromUser(referenceIntensityProfile,referencePeakFitFunction,
estimatorPromptPrefix="Reference peak:",
windowTitle="Reference Peak estimates")
except: #exception occurs if user cancels the 'dialog'
referencePeakFitFunction = None
referencePeakFitSettings = None
print 'no reference'
movingPeakFitSettings.referenceIntensityProfile = referenceIntensityProfile
referencePeakFitSettings.referenceIntensityProfile = referenceIntensityProfile
print "fitting a %s function..." % settings.defaultFitFunction
df_movingPeak = odm.calculatePeakDisplacements(df.intensityProfile, movingPeakFitSettings, factor=100,maxfev=20000)
df_movingPeak.rename(columns = lambda columnName: columnName + "_mp",inplace=True)
df = df.join(df_movingPeak)
if (referencePeakFitSettings is not None):
df_referencePeak = odm.calculatePeakDisplacements(df.intensityProfile, referencePeakFitSettings, factor=100,maxfev=20000)
df_referencePeak.rename(columns = lambda columnName: columnName + "_ref",inplace=True)
df = df.join(df_referencePeak)
df['displacement'] = df.displacement_mp - df.displacement_ref
else:
df['displacement'] = df.displacement_mp
#save settings
sys.stdout.write("saving defaults...")
globalSettings.saveToFile()
sys.stdout.write("done\r\n")
sys.stdout.write("saving local setting file...")
settings.saveToFile()
sys.stdout.write("done\r\n")
print "done"
#save calculated peak position data as csv
sys.stdout.write("saving dataframe as csv file...")
exportColumns = ['relativeTime','cycleNumber','direction','actuatorVoltage','displacement','displacement_mp','chiSquare_mp']
if ('displacement_ref' in df.columns):
exportColumns +=['displacement_ref','chiSquare_ref']
df[exportColumns].to_csv(os.path.join(commonPath,'odmanalysis.csv'),index_label='timestamp')
sys.stdout.write("done\r\n")
#save fit results as pickled dataframe
sys.stdout.write("pickling fit results dataframe as pcl file...")
fitResultColumns = ['curveFitResult_mp']
if 'curveFitResult_ref' in df.columns:
fitResultColumns += ['curveFitResult_ref']
df[fitResultColumns].to_pickle(commonPath + '/fitResults.pcl')
sys.stdout.write("done\r\n")
#save the used fit functions and fit settings as pickled objects
sys.stdout.write("pickling fit functions and settings...")
settingsDict = {'movingPeakFitSettings': movingPeakFitSettings,
'referencePeakFitSettings': referencePeakFitSettings if referencePeakFitSettings is not None else None}
with file(commonPath+'/fitSettings.pcl','w') as stream:
pickle.dump(settingsDict,stream)
sys.stdout.write("done\r\n")
print "ALL DONE"
return df,movingPeakFitSettings,referencePeakFitSettings,measurementName
def readAsync(outputQueue,inputFile):
reader = odm.getODMDataReader(inputFile)
for chunk in reader:
outputQueue.put(chunk)
