def testSpreadsheetDataCompare5(self):
sp = SpreadsheetData(data=[[i * 0.5, 1] for i in range(10)],
names=['t', 'val'])
diff = sp.compare(sp, "val")
self.assertAlmostEqual(diff["max"], 0)
self.assertAlmostEqual(diff["average"], 0)
self.assertAlmostEqual(diff["wAverage"], 0)
def __init__(self,
timeName=None,
validData=None,
validMatchRegexp=False,
csvName=None,
txtName=None,
excelName=None,
namePrefix=None,
skip_header=0,
stripCharacters=None,
progressTemplate=None,
replaceFirstLine=None,
startTime=None,
endTime=None):
self.__data=SpreadsheetData(timeName=timeName,
validData=validData,
validMatchRegexp=validMatchRegexp,
csvName=csvName,
txtName=txtName,
excelName=excelName,
skip_header=skip_header,
stripCharacters=stripCharacters,
replaceFirstLine=replaceFirstLine)
self.namePrefix=namePrefix
GeneralLineAnalyzer.__init__(self,
titles=[self.makeName(n) for n in self.__data.names(withTime=False)],
doTimelines=True,
doFiles=False,
progressTemplate=progressTemplate,
startTime=startTime,
endTime=endTime)
self.timeNew=False
self.tRange=self.__data.tRange()
def testSpreadsheetReadFileTest(self):
fName="/tmp/testdata"
open(fName,"w").write(filecontent)
sp=SpreadsheetData(txtName=fName)
self.assertEqual(len(sp.names()),4)
self.assertEqual(sp.size(),5)
self.assertEqual(sp.tRange(),(0.005,0.025))
self.assertEqual(sp.names()[1],"Initial")
def testSpreadsheetDataCompare4c(self):
sp=SpreadsheetData(data=[[i*0.5,math.sin(i)] for i in range(10)],names=['t','val'])
sp2=SpreadsheetData(data=[[i*0.5,math.sin(i)+0.5] for i in range(10)],names=['t','val'])
diff1=sp.compare(sp2,"val")
diff2=sp2.compare(sp,"val")
self.assertAlmostEqual(diff1["max"],diff2["max"])
self.assertAlmostEqual(diff1["average"],diff2["average"])
self.assertAlmostEqual(diff1["wAverage"],diff2["wAverage"])
self.assertAlmostEqual(diff1["average"],diff1["wAverage"])
def testSpreadsheetDataCompare4f(self):
sp=SpreadsheetData(data=[[i*0.5,i*0.5] for i in range(10)],names=['t','val'])
sp2=SpreadsheetData(data=[[i*0.5-0.25,i*0.5-0.25] for i in range(10)],names=['t','val'])
diff3=sp.compare(sp2,"val",common=True)
diff4=sp2.compare(sp,"val",common=True)
self.assertAlmostEqual(diff3["max"],diff4["max"])
self.assertAlmostEqual(diff3["average"],diff4["average"])
self.assertAlmostEqual(diff3["wAverage"],diff4["wAverage"])
self.assertAlmostEqual(diff3["average"],diff3["wAverage"])
self.assertAlmostEqual(diff4["average"],diff4["wAverage"])
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=self.parser.getArgs()[0:-1]
data=SpreadsheetData(txtName=sources[0],
timeName=self.opts.time,
validData=self.opts.columns,
title=path.splitext(path.basename(sources[0]))[0])
if self.opts.time==None:
self.opts.time=data.names()[0]
for s in sources[1:]:
addition=path.splitext(path.basename(s))[0]
sData=SpreadsheetData(txtName=s)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or n in self.opts.columns):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData)
data.append(addition+" "+n,d)
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def testSpreadsheetDataCompare3(self):
sp=SpreadsheetData(data=data6,names=names3)
sp2=SpreadsheetData(data=data7,names=names4)
diff1=sp.compare(sp2,"val")
self.assertAlmostEqual(diff1["max"],0)
self.assertAlmostEqual(diff1["average"],0)
self.assertAlmostEqual(diff1["wAverage"],0)
diff2=sp2.compare(sp,"val")
self.assertAlmostEqual(diff2["max"],0.5)
self.assertAlmostEqual(diff2["average"],0.09090909)
self.assertAlmostEqual(diff2["wAverage"],0.05)
def testSpreadsheetDataCompare2(self):
sp=SpreadsheetData(data=data3,names=names3)
sp2=SpreadsheetData(data=data5,names=names4)
diff1=sp.compare(sp2,"val")
self.assertAlmostEqual(diff1["max"],0.25)
self.assertAlmostEqual(diff1["average"],0.25)
self.assertAlmostEqual(diff1["wAverage"],0.25)
diff2=sp2.compare(sp,"val")
self.assertAlmostEqual(diff2["max"],9.25)
self.assertAlmostEqual(diff2["average"],1.0681818181)
self.assertAlmostEqual(diff2["wAverage"],0.7)
def testGetSpreadsheet(self):
csvName="/tmp/nix.csv"
sd=TimelineDirectory(theDir)
st=sd["p"]
spread=st()
spread.writeCSV(csvName)
self.assertEqual(len(st.positions)+1,len(spread.names()))
rereadSpread=SpreadsheetData(csvName=csvName)
self.assertEqual(len(spread.names()),len(rereadSpread.names()))
self.assertEqual(len(spread.data),len(rereadSpread.data))
sd=TimelineDirectory(theDir,"timeline")
st=sd["p"]
spread=st()
spread.writeCSV("/tmp/nix2.csv")
def testGetSpreadsheet(self):
csvName = path.join(self.theDir, "nix.csv")
sd = TimelineDirectory(self.theDir)
st = sd["p"]
spread = st()
spread.writeCSV(csvName)
self.assertEqual(len(st.positions) + 1, len(spread.names()))
rereadSpread = SpreadsheetData(csvName=csvName)
self.assertEqual(len(spread.names()), len(rereadSpread.names()))
self.assertEqual(len(spread.data), len(rereadSpread.data))
sd = TimelineDirectory(self.theDir, "timeline")
st = sd["p"]
spread = st()
spread.writeCSV(path.join(self.theDir, "nix2.csv"))
def testSpreadsheetDataCompare1(self):
sp=SpreadsheetData(data=data3,names=names3)
sp2=SpreadsheetData(data=data4,names=names4)
diff1=sp.compare(sp2,"val")
self.assertAlmostEqual(diff1["max"],1)
self.assertAlmostEqual(diff1["average"],1)
self.assertAlmostEqual(diff1["wAverage"],1)
diff2=sp2.compare(sp,"val")
self.assertEqual(diff1["max"],diff2["max"])
self.assertEqual(diff1["average"],diff2["average"])
self.assertEqual(diff1["wAverage"],diff2["wAverage"])
diff3=sp.compare(sp,"val")
self.assertAlmostEqual(diff3["max"],0)
self.assertAlmostEqual(diff3["average"],0)
self.assertAlmostEqual(diff3["wAverage"],0)
def testGetSpreadsheet(self):
csvName = path.join(self.theDir, "nix.csv")
sd = TimelineDirectory(self.theDir)
st = sd["p"]
spread = st()
assert len(spread.names()) == 5
assert spread.size() == 3
spread.writeCSV(csvName)
assert len(st.positions) + 1 == len(spread.names())
rereadSpread = SpreadsheetData(csvName=csvName)
assert len(spread.names()) == len(rereadSpread.names())
assert len(spread.data) == len(rereadSpread.data)
sd = TimelineDirectory(self.theDir, "timeline")
st = sd["p"]
spread = st()
spread.writeCSV(path.join(self.theDir, "nix2.csv"))
def testGetSpreadsheet(self):
csvName=path.join(self.theDir,"nix.csv")
sd=TimelineDirectory(self.theDir)
st=sd["p"]
spread=st()
assert len(spread.names())==5
assert spread.size()==3
spread.writeCSV(csvName)
assert len(st.positions)+1==len(spread.names())
rereadSpread=SpreadsheetData(csvName=csvName)
assert len(spread.names())==len(rereadSpread.names())
assert len(spread.data)==len(rereadSpread.data)
sd=TimelineDirectory(self.theDir,"timeline")
st=sd["p"]
spread=st()
spread.writeCSV(path.join(self.theDir,"nix2.csv"))
def testSpreadsheetScaleData(self):
sp1 = SpreadsheetData(data=data1, names=names1)
self.assertAlmostEqual(min(sp1.data["t"]), 0)
self.assertAlmostEqual(max(sp1.data["t"]), 5.0)
self.assertAlmostEqual(min(sp1.data["p1"]), 0)
self.assertAlmostEqual(max(sp1.data["p1"]), 10.0)
self.assertAlmostEqual(min(sp1.data["p2"]), 0)
self.assertAlmostEqual(max(sp1.data["p2"]), 15.0)
sp1.recalcData("t", "t/60")
self.assertAlmostEqual(min(sp1.data["t"]), 0)
self.assertAlmostEqual(max(sp1.data["t"]), 5.0 / 60.)
sp1.recalcData("p 3", "p1+p2", create=True)
self.assertAlmostEqual(min(sp1.data["p 3"]), 0)
self.assertAlmostEqual(max(sp1.data["p 3"]), 25.0)
sp1.recalcData("p4", "data['p1']+data['p2']", create=True)
self.assertAlmostEqual(min(sp1.data["p4"]), 0)
self.assertAlmostEqual(max(sp1.data["p4"]), 25.0)
sp1.recalcData("p1", "this/10+1")
self.assertAlmostEqual(min(sp1.data["p1"]), 1)
self.assertAlmostEqual(max(sp1.data["p1"]), 2)
def testSpreadsheetDataInterpolateNoName(self):
sp = SpreadsheetData(data=data3a, names=names3a)
data = sp(-1, invalidExtend=True)
self.assertEqual(len(data), 2)
self.assertAlmostEqual(data["val"], 0)
self.assertEqual(data["descr"], b('val_0'))
data = sp(2, invalidExtend=True)
self.assertAlmostEqual(data["val"], 4)
self.assertEqual(data["descr"], b('val_2'))
data = sp(5.1, invalidExtend=True)
self.assertAlmostEqual(data["val"], 26.1)
self.assertEqual(data["descr"], b('val_5'))
def testSpreadsheetReadFileTest(self):
fName = mktemp()
open(fName, "wb").write(filecontent)
sp = SpreadsheetData(txtName=fName)
self.assertEqual(len(sp.names()), 4)
self.assertEqual(sp.size(), 5)
self.assertEqual(sp.tRange(), (0.005, 0.025))
self.assertEqual(sp.names()[1], "Initial")
def testSpreadsheetScaleData(self):
sp1=SpreadsheetData(data=data1,names=names1)
self.assertAlmostEqual(min(sp1.data["t"]),0)
self.assertAlmostEqual(max(sp1.data["t"]),5.0)
self.assertAlmostEqual(min(sp1.data["p1"]),0)
self.assertAlmostEqual(max(sp1.data["p1"]),10.0)
self.assertAlmostEqual(min(sp1.data["p2"]),0)
self.assertAlmostEqual(max(sp1.data["p2"]),15.0)
sp1.recalcData("t","t/60")
self.assertAlmostEqual(min(sp1.data["t"]),0)
self.assertAlmostEqual(max(sp1.data["t"]),5.0/60.)
sp1.recalcData("p 3","p1+p2",create=True)
self.assertAlmostEqual(min(sp1.data["p 3"]),0)
self.assertAlmostEqual(max(sp1.data["p 3"]),25.0)
sp1.recalcData("p4","data['p1']+data['p2']",create=True)
self.assertAlmostEqual(min(sp1.data["p4"]),0)
self.assertAlmostEqual(max(sp1.data["p4"]),25.0)
sp1.recalcData("p1","this/10+1")
self.assertAlmostEqual(min(sp1.data["p1"]),1)
self.assertAlmostEqual(max(sp1.data["p1"]),2)
def testSpreadsheetDataInterpolation(self):
sp = SpreadsheetData(data=data3, names=names3)
self.assertAlmostEqual(sp(-1, "val", invalidExtend=True), 0)
self.assert_(numpy.isnan(sp(-1, "val")))
self.assertAlmostEqual(sp(10, "val", invalidExtend=True), 81)
self.assert_(numpy.isnan(sp(10, "val")))
self.assertAlmostEqual(sp(1, "val"), 1)
self.assertAlmostEqual(sp(1.5, "val"), 2.5)
self.assertAlmostEqual(sp(5, "val"), 25)
self.assertAlmostEqual(sp(5.1, "val"), 26.1)
self.assertAlmostEqual(sp(8.9, "val"), 79.3)
self.assertAlmostEqual(sp(8.9, "t"), 8.9)
self.assertAlmostEqual(sp(1, "val", noInterpolation=True), 1)
self.assert_(numpy.isnan(sp(1.5, "val", noInterpolation=True)))
def testSpreadsheetDataCompare3(self):
sp = SpreadsheetData(data=data6, names=names3)
sp2 = SpreadsheetData(data=data7, names=names4)
diff1 = sp.compare(sp2, "val")
self.assertAlmostEqual(diff1["max"], 0)
self.assertAlmostEqual(diff1["average"], 0)
self.assertAlmostEqual(diff1["wAverage"], 0)
diff2 = sp2.compare(sp, "val")
self.assertAlmostEqual(diff2["max"], 0.5)
self.assertAlmostEqual(diff2["average"], 0.09090909)
self.assertAlmostEqual(diff2["wAverage"], 0.05)
def testSpreadsheetDataCompare4b(self):
sp = SpreadsheetData(data=[[i * 0.5, 1 + i * i] for i in range(10)],
names=['t', 'val'])
sp2 = SpreadsheetData(data=[[i * 0.5, 2 + i * i] for i in range(10)],
names=['t', 'val'])
diff1 = sp.compare(sp2, "val")
diff2 = sp2.compare(sp, "val")
self.assertAlmostEqual(diff1["max"], diff2["max"])
self.assertAlmostEqual(diff1["average"], diff2["average"])
self.assertAlmostEqual(diff1["wAverage"], diff2["wAverage"])
self.assertAlmostEqual(diff1["average"], diff1["wAverage"])
def testSpreadsheetDataCompare2(self):
sp = SpreadsheetData(data=data3, names=names3)
sp2 = SpreadsheetData(data=data5, names=names4)
diff1 = sp.compare(sp2, "val")
self.assertAlmostEqual(diff1["max"], 0.25)
self.assertAlmostEqual(diff1["average"], 0.25)
self.assertAlmostEqual(diff1["wAverage"], 0.25)
diff2 = sp2.compare(sp, "val")
self.assertAlmostEqual(diff2["max"], 9.25)
self.assertAlmostEqual(diff2["average"], 1.0681818181)
self.assertAlmostEqual(diff2["wAverage"], 0.7)
def testSpreadsheetDataCompare1(self):
sp = SpreadsheetData(data=data3, names=names3)
sp2 = SpreadsheetData(data=data4, names=names4)
diff1 = sp.compare(sp2, "val")
self.assertAlmostEqual(diff1["max"], 1)
self.assertAlmostEqual(diff1["average"], 1)
self.assertAlmostEqual(diff1["wAverage"], 1)
diff2 = sp2.compare(sp, "val")
self.assertEqual(diff1["max"], diff2["max"])
self.assertEqual(diff1["average"], diff2["average"])
self.assertEqual(diff1["wAverage"], diff2["wAverage"])
diff3 = sp.compare(sp, "val")
self.assertAlmostEqual(diff3["max"], 0)
self.assertAlmostEqual(diff3["average"], 0)
self.assertAlmostEqual(diff3["wAverage"], 0)
def testSpreadsheetDataCompare4f(self):
sp = SpreadsheetData(data=[[i * 0.5, i * 0.5] for i in range(10)],
names=['t', 'val'])
sp2 = SpreadsheetData(data=[[i * 0.5 - 0.25, i * 0.5 - 0.25]
for i in range(10)],
names=['t', 'val'])
diff3 = sp.compare(sp2, "val", common=True)
diff4 = sp2.compare(sp, "val", common=True)
self.assertAlmostEqual(diff3["max"], diff4["max"])
self.assertAlmostEqual(diff3["average"], diff4["average"])
self.assertAlmostEqual(diff3["wAverage"], diff4["wAverage"])
self.assertAlmostEqual(diff3["average"], diff3["wAverage"])
self.assertAlmostEqual(diff4["average"], diff4["wAverage"])
def testSpreadsheetDataCompare4d(self):
sp = SpreadsheetData(data=[[i * 0.5, math.sin(i)] for i in range(10)],
names=['t', 'val'])
sp2 = SpreadsheetData(data=[[i * 0.5 - 0.5,
math.sin(i - 1) + 0.5]
for i in range(11)],
names=['t', 'val'])
diff1 = sp.compare(sp2, "val")
diff2 = sp2.compare(sp, "val")
self.assertAlmostEqual(diff1["max"], diff2["max"])
# self.assertAlmostEqual(diff1["average"],diff2["average"])
# self.assertAlmostEqual(diff1["wAverage"],diff2["wAverage"])
self.assertAlmostEqual(diff1["average"], diff1["wAverage"])
# self.assertAlmostEqual(diff2["average"],diff2["wAverage"])
diff3 = sp.compare(sp2, "val", common=True)
diff4 = sp2.compare(sp, "val", common=True)
self.assertAlmostEqual(diff3["max"], diff4["max"])
self.assertAlmostEqual(diff3["average"], diff4["average"])
self.assertAlmostEqual(diff3["wAverage"], diff4["wAverage"])
self.assertAlmostEqual(diff3["average"], diff3["wAverage"])
self.assertAlmostEqual(diff4["average"], diff4["wAverage"])
def testSpreadsheetDataCompare6(self):
sp = SpreadsheetData(data=[[i * 0.5, 1] for i in range(10)],
names=['t', 'val'])
sp2 = SpreadsheetData(data=[[i * 0.5 + 10, 2] for i in range(10)],
names=['t', 'val'])
diff1 = sp.compare(sp2, "val")
diff2 = sp2.compare(sp, "val")
self.assertAlmostEqual(diff1["max"], 1)
self.assertAlmostEqual(diff1["average"], 1)
self.assertAlmostEqual(diff1["wAverage"], 1)
self.assertAlmostEqual(diff2["max"], 1)
self.assertAlmostEqual(diff2["average"], 1)
self.assertAlmostEqual(diff2["wAverage"], 1)
diff1 = sp.compare(sp2, "val", common=True)
diff2 = sp2.compare(sp, "val", common=True)
self.assertEqual(diff1["max"], None)
self.assertEqual(diff1["average"], None)
self.assertEqual(diff1["wAverage"], None)
self.assertEqual(diff2["max"], None)
self.assertEqual(diff2["average"], None)
self.assertEqual(diff2["wAverage"], None)
def __call__(self):
"""Return the data as a SpreadsheetData-object"""
lines = open(self.file).readlines()
data = []
for l in lines:
v = l.split()
if v[0][0] != '#':
data.append(
[float(x.replace('(', '').replace(')', '')) for x in v])
names = ["time"]
if self.isVector:
for p in self.positions:
names += [p + " x", p + " y", p + " z"]
else:
names += self.positions
return SpreadsheetData(data=data,
names=names,
title="%s_t=%s" % (self.val, self.time))
def testSpreadsheetDataInterpolationWithString(self):
sp = SpreadsheetData(data=data3a, names=names3a)
self.assertAlmostEqual(sp(-1, "val", invalidExtend=True), 0)
self.assert_(numpy.isnan(sp(-1, "val")))
self.assertAlmostEqual(sp(10, "val", invalidExtend=True), 81)
self.assert_(numpy.isnan(sp(10, "val")))
self.assertAlmostEqual(sp(1, "val"), 1)
self.assertAlmostEqual(sp(1.5, "val"), 2.5)
self.assertAlmostEqual(sp(5, "val"), 25)
self.assertAlmostEqual(sp(5.1, "val"), 26.1)
self.assertAlmostEqual(sp(8.9, "val"), 79.3)
self.assertAlmostEqual(sp(8.9, "t"), 8.9)
self.assertAlmostEqual(sp(1, "val", noInterpolation=True), 1)
self.assert_(numpy.isnan(sp(1.5, "val", noInterpolation=True)))
self.assertEqual(sp(-1, "descr", invalidExtend=True), b('val_0'))
self.assertEqual(sp(-1, "descr"), '')
self.assertEqual(sp(2, "descr"), b('val_2'))
self.assertEqual(sp(3, "descr"), b('val_3'))
self.assertEqual(sp(2.4, "descr"), b('val_2'))
self.assertEqual(sp(2.5, "descr"), b('val_3'))
self.assertEqual(sp(2.7, "descr"), b('val_3'))
def __call__(self, scaleX=1., scaleData=1, offsetData=0, offsetX=0):
"""Return the data as SpreadsheetData-object"""
data = []
if self.isVector():
for i, c in enumerate(self.coord):
data.append([scaleX * c + offsetX] +
[scaleData * v + offsetData for v in self.data[i]])
else:
for i, c in enumerate(self.coord):
data.append([
scaleX * c + offsetX, scaleData * self.data[i] + offsetData
])
names = ["coord"]
if self.isVector():
names += [self.name + "_x", self.name + "_y", self.name + "_z"]
else:
names.append(self.name)
return SpreadsheetData(data=data,
names=names,
title="%s_t=%s" % (self.line(), self.time()))
def __call__(self,addTitle=True):
"""Return the data as a SpreadsheetData-object"""
def safeFloat(v):
try:
return float(v)
except ValueError:
return v
lines=open(self.file).readlines()
data=[]
for l in lines:
v=l.split()
if v[0][0]!='#':
data.append([safeFloat(x.replace('(','').replace(')','')) for x in v])
names=["time"]
if self.isVector:
for p in self.positions:
names+=[p+" x",p+" y",p+" z"]
else:
names+=self.positions
return SpreadsheetData(data=data,
names=names,
title="%s_t=%s" % (self.val,self.time) if addTitle else None)
def testSpreadsheetDataCompare6(self):
sp=SpreadsheetData(data=[[i*0.5,1] for i in range(10)],names=['t','val'])
sp2=SpreadsheetData(data=[[i*0.5+10,2] for i in range(10)],names=['t','val'])
diff1=sp.compare(sp2,"val")
diff2=sp2.compare(sp,"val")
self.assertAlmostEqual(diff1["max"],1)
self.assertAlmostEqual(diff1["average"],1)
self.assertAlmostEqual(diff1["wAverage"],1)
self.assertAlmostEqual(diff2["max"],1)
self.assertAlmostEqual(diff2["average"],1)
self.assertAlmostEqual(diff2["wAverage"],1)
diff1=sp.compare(sp2,"val",common=True)
diff2=sp2.compare(sp,"val",common=True)
self.assertEqual(diff1["max"],None)
self.assertEqual(diff1["average"],None)
self.assertEqual(diff1["wAverage"],None)
self.assertEqual(diff2["max"],None)
self.assertEqual(diff2["average"],None)
self.assertEqual(diff2["wAverage"],None)
def testSpreadsheetDataAddTimes(self):
sp1 = SpreadsheetData(data=data1, names=names1)
self.assertEqual(len(sp1.data), 6)
sp1.addTimes([1.5], interpolate=True)
self.assertEqual(len(sp1.data), 7)
self.assertAlmostEqual(sp1.data["p1"][2], 3)
sp1.addTimes([2.5], interpolate=False)
self.assertEqual(len(sp1.data), 8)
self.assert_(numpy.isnan(sp1.data["p1"][4]))
sp1.addTimes([-1], interpolate=True, invalidExtend=True)
self.assertEqual(len(sp1.data), 9)
self.assertAlmostEqual(sp1.data["p1"][0], 0)
sp1.addTimes([10], interpolate=True, invalidExtend=False)
self.assertEqual(len(sp1.data), 10)
self.assert_(numpy.isnan(sp1.data["p1"][-1]))
sp1.addTimes([0, 1, 2], interpolate=True)
self.assertEqual(len(sp1.data), 10)
sp1.addTimes([-2, 2.37, 20], interpolate=True)
self.assertEqual(len(sp1.data), 13)
sp1.addTimes([-3, 4, 19], interpolate=True)
self.assertEqual(len(sp1.data), 15)
def testSpreadsheetReadFileHandleTest(self):
sp = SpreadsheetData(txtName=BytesIO(filecontent))
self.assertEqual(len(sp.names()), 4)
def testSpreadsheetDataExtend(self):
sp1 = SpreadsheetData(data=data1, names=names1)
sp1.append("test", [i * i for i in range(len(data1))])
self.assertEqual(len(sp1.names()), len(names1) + 1)
self.assert_("test" in sp1.names())
def testSpreadsheetDataExtend(self):
sp1=SpreadsheetData(data=data1,names=names1)
sp1.append("test",[i*i for i in range(len(data1))])
self.assertEqual(len(sp1.names()),len(names1)+1)
self.assert_("test" in sp1.names())
def testSpreadsheetReadFileHandleTest(self):
fName = mktemp()
open(fName, "w").write(filecontent)
sp = SpreadsheetData(txtName=open(fName))
self.assertEqual(len(sp.names()), 4)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest, "exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(csvName=sources[0], title=path.splitext(path.basename(sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(csvName=s)
if self.opts.addTimes:
data.addTimes(
time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime
)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate,
)
data.append(addition + " " + n, d, allowDuplicates=True)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def testSpreadsheetReadFileHandleTest(self):
fName="/tmp/testdata"
open(fName,"w").write(filecontent)
sp=SpreadsheetData(txtName=open(fName))
self.assertEqual(len(sp.names()),4)
def testSpreadsheetDataConstruction(self):
sp=SpreadsheetData(data=data1,names=names1)
self.assertEqual(names1,list(sp.names()))
self.assertEqual(len(sp.data),len(names1)*2)
self.assertEqual(max(sp.data['t']-range(len(names1)*2)),0)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(csvName=sources[0],
title=path.splitext(path.basename(
sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(csvName=s)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition + " " + n, d, allowDuplicates=True)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(txtName=sources[0],
timeName=self.opts.time,
validData=self.opts.columns,
title=path.splitext(path.basename(
sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(txtName=s)
for n in sData.names():
if n != self.opts.time and (self.opts.columns == []
or n in self.opts.columns):
d = data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData)
data.append(addition + " " + n, d)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def testSpreadsheetDataAddition(self):
sp1 = SpreadsheetData(data=data1, names=names1)
sp2 = SpreadsheetData(data=data2, names=names2, title="nix")
sp = sp1 + sp2
self.assertEqual(len(names1) + len(names2) - 1, len(sp.names()))
class ReplayDataFileAnalyzer(GeneralLineAnalyzer):
"""Reads Data into a SpreadsheetData-object and then every time a
new time value comes up returns the data from that file"""
def __init__(self,
timeName=None,
validData=None,
validMatchRegexp=False,
csvName=None,
txtName=None,
excelName=None,
namePrefix=None,
skip_header=0,
stripCharacters=None,
progressTemplate=None,
replaceFirstLine=None,
startTime=None,
endTime=None):
self.__data=SpreadsheetData(timeName=timeName,
validData=validData,
validMatchRegexp=validMatchRegexp,
csvName=csvName,
txtName=txtName,
excelName=excelName,
skip_header=skip_header,
stripCharacters=stripCharacters,
replaceFirstLine=replaceFirstLine)
self.namePrefix=namePrefix
GeneralLineAnalyzer.__init__(self,
titles=[self.makeName(n) for n in self.__data.names(withTime=False)],
doTimelines=True,
doFiles=False,
progressTemplate=progressTemplate,
startTime=startTime,
endTime=endTime)
self.timeNew=False
self.tRange=self.__data.tRange()
def makeName(self,name):
if self.namePrefix is None:
return name
else:
return self.namePrefix+name
def timeChanged(self):
time=float(self.getTime())
if time<self.tRange[0] or time>self.tRange[1]:
return
self.timeNew=True
self.lines.setTime(time)
def doMatch(self,line):
if self.timeNew:
time=float(self.getTime())
if time<self.tRange[0] or time>self.tRange[1]:
return None
data=self.__data(time,invalidExtend=True)
self.timeNew=False
return {self.makeName(k):v for k,v in data.items()}
else:
return None
def addToTimelines(self,data):
for t in self.titles:
self.lines.setValue(self.fName(t),data[t])
def processProgressTemplate(self,data):
ldata={k[len(self.namePrefix):]:data[k] for k in data}
myProgress=self.progressTemplate.format(**ldata)
return myProgress
def testSpreadsheetDataConstruction(self):
sp = SpreadsheetData(data=data1, names=names1)
self.assertEqual(names1, list(sp.names()))
self.assertEqual(len(sp.data), len(names1) * 2)
self.assertEqual(max(sp.data['t'] - range(len(names1) * 2)), 0)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=self.parser.getArgs()[0:-1]
diffs=[None]
if len(sources)>1:
# find differing parts
commonStart=1e4
commonEnd=1e4
for s in sources[1:]:
a=path.abspath(sources[0])
b=path.abspath(s)
start=0
end=0
for i in range(min(len(a),len(b))):
start=i
if a[i]!=b[i]:
break
commonStart=min(commonStart,start)
for i in range(min(len(a),len(b))):
end=i
if a[-(i+1)]!=b[-(i+1)]:
break
commonEnd=min(commonEnd,end)
diffs=[]
for s in sources:
b=path.abspath(s)
if commonEnd>0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names=None
title=path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
names=path.splitext(path.basename(sources[0]))[0].split("_")
title=None
data=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.names()[0]
for i,s in enumerate(sources[1:]):
names=None
title=path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names=title.split("_")
title=None
sData=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData)
data.append(diffs[i+1]+" "+n,d)
self.joinedAddColumns(data)
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.writeExcel:
data.getData().to_excel(dest)
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def testSpreadsheetReadFileHandleTest(self):
sp=SpreadsheetData(txtName=StringIO(filecontent))
self.assertEqual(len(sp.names()),4)
def testSpreadsheetDataOuterLimitNoInterpolate(self):
sp = SpreadsheetData(data=data3, names=names3)
self.assertAlmostEqual(sp(0, "val", noInterpolation=True), 0)
self.assertAlmostEqual(sp(9, "val", noInterpolation=True), 81)
self.assertAlmostEqual(sp(0, "val"), 0)
self.assertAlmostEqual(sp(9, "val"), 81)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=self.parser.getArgs()[0:-1]
data=SpreadsheetData(title=path.splitext(path.basename(sources[0]))[0],
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(sources[0]))
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.names()[0]
for s in sources[1:]:
addition=path.splitext(path.basename(s))[0]
sData=SpreadsheetData(validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(s))
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time:
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition+" "+n,
d,
allowDuplicates=True)
self.joinedAddColumns(data)
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.writeExcel:
data.getData().to_excel(dest)
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def testSpreadsheetDataAddTimes(self):
sp1=SpreadsheetData(data=data1,names=names1)
self.assertEqual(len(sp1.data),6)
sp1.addTimes([1.5],interpolate=True)
self.assertEqual(len(sp1.data),7)
self.assertAlmostEqual(sp1.data["p1"][2],3)
sp1.addTimes([2.5],interpolate=False)
self.assertEqual(len(sp1.data),8)
self.assert_(math.isnan(sp1.data["p1"][4]))
sp1.addTimes([-1],interpolate=True,invalidExtend=True)
self.assertEqual(len(sp1.data),9)
self.assertAlmostEqual(sp1.data["p1"][0],0)
sp1.addTimes([10],interpolate=True,invalidExtend=False)
self.assertEqual(len(sp1.data),10)
self.assert_(math.isnan(sp1.data["p1"][-1]))
sp1.addTimes([0,1,2],interpolate=True)
self.assertEqual(len(sp1.data),10)
sp1.addTimes([-2,2.37,20],interpolate=True)
self.assertEqual(len(sp1.data),13)
sp1.addTimes([-3,4,19],interpolate=True)
self.assertEqual(len(sp1.data),15)
def testSpreadsheetDataCompare5(self):
sp=SpreadsheetData(data=[[i*0.5,1] for i in range(10)],names=['t','val'])
diff=sp.compare(sp,"val")
self.assertAlmostEqual(diff["max"],0)
self.assertAlmostEqual(diff["average"],0)
self.assertAlmostEqual(diff["wAverage"],0)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=[]
for s in self.parser.getArgs()[0:-1]:
if s.find("/*lastTime*/")>=0:
front,back=s.split("/*lastTime*/",1)
for d in glob(front):
lastTime=None
for f in listdir(d):
if path.exists(path.join(d,f,back)):
try:
t=float(f)
if lastTime:
if t>float(lastTime):
lastTime=f
else:
lastTime=f
except ValueError:
pass
if lastTime:
sources.append(path.join(d,lastTime,back))
else:
sources.append(s)
diffs=[None]
if len(sources)>1:
# find differing parts
commonStart=1e4
commonEnd=1e4
for s in sources[1:]:
a=path.abspath(sources[0])
b=path.abspath(s)
start=0
end=0
for i in range(min(len(a),len(b))):
start=i
if a[i]!=b[i]:
break
commonStart=min(commonStart,start)
for i in range(min(len(a),len(b))):
end=i
if a[-(i+1)]!=b[-(i+1)]:
break
commonEnd=min(commonEnd,end)
diffs=[]
for s in sources:
b=path.abspath(s)
if commonEnd>0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names=None
title=path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
names=path.splitext(path.basename(sources[0]))[0].split("_")
title=None
data=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
rawData=[deepcopy(data)]
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.timeName()
if not diffs[0] is None:
data.rename(lambda c:diffs[0]+" "+c)
for i,s in enumerate(sources[1:]):
names=None
title=path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names=title.split("_")
title=None
sData=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
rawData.append(sData)
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(diffs[i+1]+" "+n,d)
self.joinedAddColumns(data)
data.rename(self.processName,renameTime=True)
data.rename(lambda c:c.strip())
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.automaticFormat:
if self.getDataFormat(dest)=="excel":
self.opts.writeExcel=True
if self.opts.writeExcel:
from pandas import ExcelWriter
with ExcelWriter(dest) as writer:
data.getData().to_excel(writer)
if self.opts.addSheets:
for n,d in enumerate(rawData):
d.getData().to_excel(writer,
sheet_name="Original file %d" % n)
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=[]
for s in self.parser.getArgs()[0:-1]:
if s.find("/*lastTime*/")>=0:
front,back=s.split("/*lastTime*/",1)
for d in glob(front):
lastTime=None
for f in listdir(d):
if path.exists(path.join(d,f,back)):
try:
t=float(f)
if lastTime:
if t>float(lastTime):
lastTime=f
else:
lastTime=f
except ValueError:
pass
if lastTime:
sources.append(path.join(d,lastTime,back))
else:
sources.append(s)
diffs=[None]
if len(sources)>1:
# find differing parts
commonStart=1e4
commonEnd=1e4
for s in sources[1:]:
a=path.abspath(sources[0])
b=path.abspath(s)
start=0
end=0
for i in range(min(len(a),len(b))):
start=i
if a[i]!=b[i]:
break
commonStart=min(commonStart,start)
for i in range(min(len(a),len(b))):
end=i
if a[-(i+1)]!=b[-(i+1)]:
break
commonEnd=min(commonEnd,end)
diffs=[]
for s in sources:
b=path.abspath(s)
if commonEnd>0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names=self.names
title=path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
if not names is None:
self.error("Names already specified as",names,". Can't calc from filename")
names=path.splitext(path.basename(sources[0]))[0].split("_")
title=None
data=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
rawData=[deepcopy(data)]
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.timeName()
if not diffs[0] is None:
data.rename(lambda c:diffs[0]+" "+c)
for i,s in enumerate(sources[1:]):
names=None
title=path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names=title.split("_")
title=None
sData=SpreadsheetData(names=names,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
rawData.append(sData)
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(diffs[i+1]+" "+n,d)
self.joinedAddColumns(data)
data.rename(self.processName,renameTime=True)
data.rename(lambda c:c.strip())
data.eliminatedNames=None
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.automaticFormat:
if self.getDataFormat(dest)=="excel":
self.opts.writeExcel=True
if self.opts.writeExcel:
from pandas import ExcelWriter
with ExcelWriter(dest) as writer:
data.getData().to_excel(writer,sheet_name="Data")
if self.opts.addSheets:
for n,d in enumerate(rawData):
d.getData().to_excel(writer,
sheet_name="Original file %d" % n)
if hasXlsxWriter:
if len(self.opts.addFormulas)>0:
from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell
rows=len(data.getData())
sheet=writer.sheets["Data"]
cols={}
for i,n in enumerate(data.names()):
cols[n]=i
newC=i
for f in self.opts.addFormulas:
newC+=1
name,formula=f.split(":::")
sheet.write(0,newC,name)
cols[name]=newC
splitted=[]
ind=0
while ind>=0:
if ind>=len(formula):
break
nInd=formula.find("'",ind)
if nInd<0:
splitted.append(formula[ind:])
ind=nInd
elif nInd!=ind:
splitted.append(formula[ind:nInd])
ind=nInd
else:
nInd=formula.find("'",ind+1)
if nInd<0:
self.error("No closing ' in formula",formula)
name=formula[ind+1:nInd]
if name not in cols:
self.error("Name",name,"not in column names",cols.keys())
splitted.append(cols[name])
ind=nInd+1
for row in range(rows):
cellFormula="="
for s in splitted:
if type(s)==int:
cellFormula+=rowCol2Cell(row+1,s)
else:
cellFormula+=s
sheet.write(row+1,newC,cellFormula)
print_("Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9")
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(title=path.splitext(path.basename(
sources[0]))[0],
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(sources[0]))
self.printColumns(sources[0], data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(s))
self.printColumns(s, sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition + " " + n, d, allowDuplicates=True)
self.joinedAddColumns(data)
if len(sources) > 1:
self.printColumns("written data", data)
if self.opts.writeExcel:
data.getData().to_excel(dest)
else:
data.writeCSV(dest, delimiter=self.opts.delimiter)
