def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest, "exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(csvName=sources[0], title=path.splitext(path.basename(sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(csvName=s)
if self.opts.addTimes:
data.addTimes(
time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime
)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate,
)
data.append(addition + " " + n, d, allowDuplicates=True)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(csvName=sources[0],
title=path.splitext(path.basename(
sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(csvName=s)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition + " " + n, d, allowDuplicates=True)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(title=path.splitext(path.basename(
sources[0]))[0],
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(sources[0]))
self.printColumns(sources[0], data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(s))
self.printColumns(s, sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition + " " + n, d, allowDuplicates=True)
self.joinedAddColumns(data)
if len(sources) > 1:
self.printColumns("written data", data)
if self.opts.writeExcel:
data.getData().to_excel(dest)
else:
data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=self.parser.getArgs()[0:-1]
data=SpreadsheetData(title=path.splitext(path.basename(sources[0]))[0],
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(sources[0]))
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.names()[0]
for s in sources[1:]:
addition=path.splitext(path.basename(s))[0]
sData=SpreadsheetData(validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
**self.dataFormatOptions(s))
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time:
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(addition+" "+n,
d,
allowDuplicates=True)
self.joinedAddColumns(data)
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.writeExcel:
data.getData().to_excel(dest)
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=[]
for s in self.parser.getArgs()[0:-1]:
if s.find("/*lastTime*/")>=0:
front,back=s.split("/*lastTime*/",1)
for d in glob(front):
lastTime=None
for f in listdir(d):
if path.exists(path.join(d,f,back)):
try:
t=float(f)
if lastTime:
if t>float(lastTime):
lastTime=f
else:
lastTime=f
except ValueError:
pass
if lastTime:
sources.append(path.join(d,lastTime,back))
else:
sources.append(s)
diffs=[None]
if len(sources)>1:
# find differing parts
commonStart=1e4
commonEnd=1e4
for s in sources[1:]:
a=path.abspath(sources[0])
b=path.abspath(s)
start=0
end=0
for i in range(min(len(a),len(b))):
start=i
if a[i]!=b[i]:
break
commonStart=min(commonStart,start)
for i in range(min(len(a),len(b))):
end=i
if a[-(i+1)]!=b[-(i+1)]:
break
commonEnd=min(commonEnd,end)
diffs=[]
for s in sources:
b=path.abspath(s)
if commonEnd>0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names=self.names
title=path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
if not names is None:
self.error("Names already specified as",names,". Can't calc from filename")
names=path.splitext(path.basename(sources[0]))[0].split("_")
title=None
data=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
rawData=[deepcopy(data)]
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.timeName()
if not diffs[0] is None:
data.rename(lambda c:diffs[0]+" "+c)
for i,s in enumerate(sources[1:]):
names=None
title=path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names=title.split("_")
title=None
sData=SpreadsheetData(names=names,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
rawData.append(sData)
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(diffs[i+1]+" "+n,d)
self.joinedAddColumns(data)
data.rename(self.processName,renameTime=True)
data.rename(lambda c:c.strip())
data.eliminatedNames=None
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.automaticFormat:
if self.getDataFormat(dest)=="excel":
self.opts.writeExcel=True
if self.opts.writeExcel:
from pandas import ExcelWriter
with ExcelWriter(dest) as writer:
data.getData().to_excel(writer,sheet_name="Data")
if self.opts.addSheets:
for n,d in enumerate(rawData):
d.getData().to_excel(writer,
sheet_name="Original file %d" % n)
if hasXlsxWriter:
if len(self.opts.addFormulas)>0:
from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell
rows=len(data.getData())
sheet=writer.sheets["Data"]
cols={}
for i,n in enumerate(data.names()):
cols[n]=i
newC=i
for f in self.opts.addFormulas:
newC+=1
name,formula=f.split(":::")
sheet.write(0,newC,name)
cols[name]=newC
splitted=[]
ind=0
while ind>=0:
if ind>=len(formula):
break
nInd=formula.find("'",ind)
if nInd<0:
splitted.append(formula[ind:])
ind=nInd
elif nInd!=ind:
splitted.append(formula[ind:nInd])
ind=nInd
else:
nInd=formula.find("'",ind+1)
if nInd<0:
self.error("No closing ' in formula",formula)
name=formula[ind+1:nInd]
if name not in cols:
self.error("Name",name,"not in column names",cols.keys())
splitted.append(cols[name])
ind=nInd+1
for row in range(rows):
cellFormula="="
for s in splitted:
if type(s)==int:
cellFormula+=rowCol2Cell(row+1,s)
else:
cellFormula+=s
sheet.write(row+1,newC,cellFormula)
print_("Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9")
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def testSpreadsheetDataAddTimes(self):
sp1 = SpreadsheetData(data=data1, names=names1)
self.assertEqual(len(sp1.data), 6)
sp1.addTimes([1.5], interpolate=True)
self.assertEqual(len(sp1.data), 7)
self.assertAlmostEqual(sp1.data["p1"][2], 3)
sp1.addTimes([2.5], interpolate=False)
self.assertEqual(len(sp1.data), 8)
self.assert_(numpy.isnan(sp1.data["p1"][4]))
sp1.addTimes([-1], interpolate=True, invalidExtend=True)
self.assertEqual(len(sp1.data), 9)
self.assertAlmostEqual(sp1.data["p1"][0], 0)
sp1.addTimes([10], interpolate=True, invalidExtend=False)
self.assertEqual(len(sp1.data), 10)
self.assert_(numpy.isnan(sp1.data["p1"][-1]))
sp1.addTimes([0, 1, 2], interpolate=True)
self.assertEqual(len(sp1.data), 10)
sp1.addTimes([-2, 2.37, 20], interpolate=True)
self.assertEqual(len(sp1.data), 13)
sp1.addTimes([-3, 4, 19], interpolate=True)
self.assertEqual(len(sp1.data), 15)
def testSpreadsheetDataAddTimes(self):
sp1=SpreadsheetData(data=data1,names=names1)
self.assertEqual(len(sp1.data),6)
sp1.addTimes([1.5],interpolate=True)
self.assertEqual(len(sp1.data),7)
self.assertAlmostEqual(sp1.data["p1"][2],3)
sp1.addTimes([2.5],interpolate=False)
self.assertEqual(len(sp1.data),8)
self.assert_(math.isnan(sp1.data["p1"][4]))
sp1.addTimes([-1],interpolate=True,invalidExtend=True)
self.assertEqual(len(sp1.data),9)
self.assertAlmostEqual(sp1.data["p1"][0],0)
sp1.addTimes([10],interpolate=True,invalidExtend=False)
self.assertEqual(len(sp1.data),10)
self.assert_(math.isnan(sp1.data["p1"][-1]))
sp1.addTimes([0,1,2],interpolate=True)
self.assertEqual(len(sp1.data),10)
sp1.addTimes([-2,2.37,20],interpolate=True)
self.assertEqual(len(sp1.data),13)
sp1.addTimes([-3,4,19],interpolate=True)
self.assertEqual(len(sp1.data),15)
def run(self):
dest=self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file",dest,"exists already. Use --force to overwrite")
sources=[]
for s in self.parser.getArgs()[0:-1]:
if s.find("/*lastTime*/")>=0:
front,back=s.split("/*lastTime*/",1)
for d in glob(front):
lastTime=None
for f in listdir(d):
if path.exists(path.join(d,f,back)):
try:
t=float(f)
if lastTime:
if t>float(lastTime):
lastTime=f
else:
lastTime=f
except ValueError:
pass
if lastTime:
sources.append(path.join(d,lastTime,back))
else:
sources.append(s)
diffs=[None]
if len(sources)>1:
# find differing parts
commonStart=1e4
commonEnd=1e4
for s in sources[1:]:
a=path.abspath(sources[0])
b=path.abspath(s)
start=0
end=0
for i in range(min(len(a),len(b))):
start=i
if a[i]!=b[i]:
break
commonStart=min(commonStart,start)
for i in range(min(len(a),len(b))):
end=i
if a[-(i+1)]!=b[-(i+1)]:
break
commonEnd=min(commonEnd,end)
diffs=[]
for s in sources:
b=path.abspath(s)
if commonEnd>0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names=None
title=path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
names=path.splitext(path.basename(sources[0]))[0].split("_")
title=None
data=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
rawData=[deepcopy(data)]
self.printColumns(sources[0],data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time==None:
self.opts.time=data.timeName()
if not diffs[0] is None:
data.rename(lambda c:diffs[0]+" "+c)
for i,s in enumerate(sources[1:]):
names=None
title=path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names=title.split("_")
title=None
sData=SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
rawData.append(sData)
self.printColumns(s,sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)):
d=data.resample(sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(diffs[i+1]+" "+n,d)
self.joinedAddColumns(data)
data.rename(self.processName,renameTime=True)
data.rename(lambda c:c.strip())
if len(sources)>1:
self.printColumns("written data",data)
if self.opts.automaticFormat:
if self.getDataFormat(dest)=="excel":
self.opts.writeExcel=True
if self.opts.writeExcel:
from pandas import ExcelWriter
with ExcelWriter(dest) as writer:
data.getData().to_excel(writer)
if self.opts.addSheets:
for n,d in enumerate(rawData):
d.getData().to_excel(writer,
sheet_name="Original file %d" % n)
else:
data.writeCSV(dest,
delimiter=self.opts.delimiter)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest,
"exists already. Use --force to overwrite")
sources = []
for s in self.parser.getArgs()[0:-1]:
if s.find("/*lastTime*/") >= 0:
front, back = s.split("/*lastTime*/", 1)
for d in glob(front):
lastTime = None
for f in listdir(d):
if path.exists(path.join(d, f, back)):
try:
t = float(f)
if lastTime:
if t > float(lastTime):
lastTime = f
else:
lastTime = f
except ValueError:
pass
if lastTime:
sources.append(path.join(d, lastTime, back))
else:
sources.append(s)
diffs = [None]
if len(sources) > 1:
# find differing parts
commonStart = 1e4
commonEnd = 1e4
for s in sources[1:]:
a = path.abspath(sources[0])
b = path.abspath(s)
start = 0
end = 0
for i in range(min(len(a), len(b))):
start = i
if a[i] != b[i]:
break
commonStart = min(commonStart, start)
for i in range(min(len(a), len(b))):
end = i
if a[-(i + 1)] != b[-(i + 1)]:
break
commonEnd = min(commonEnd, end)
diffs = []
for s in sources:
b = path.abspath(s)
if commonEnd > 0:
diffs.append(b[commonStart:-(commonEnd)])
else:
diffs.append(b[commonStart:])
names = self.names
title = path.splitext(path.basename(sources[0]))[0]
if self.opts.namesFromFilename:
if not names is None:
self.error("Names already specified as", names,
". Can't calc from filename")
names = path.splitext(path.basename(sources[0]))[0].split("_")
title = None
data = SpreadsheetData(names=names,
timeName=self.opts.time,
validData=self.opts.columns,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(sources[0]))
rawData = [deepcopy(data)]
self.printColumns(sources[0], data)
self.recalcColumns(data)
self.rawAddColumns(data)
if self.opts.time == None:
self.opts.time = data.timeName()
if not diffs[0] is None:
data.rename(lambda c: diffs[0] + " " + c)
for i, s in enumerate(sources[1:]):
names = None
title = path.splitext(path.basename(s))[0]
if self.opts.namesFromFilename:
names = title.split("_")
title = None
sData = SpreadsheetData(
names=names,
skip_header=self.opts.skipHeaderLines,
stripCharacters=self.opts.stripCharacters,
replaceFirstLine=self.opts.replaceFirstLine,
timeName=self.opts.time,
validData=self.opts.columns,
validMatchRegexp=self.opts.columnsRegexp,
title=title,
**self.dataFormatOptions(s))
rawData.append(sData)
self.printColumns(s, sData)
self.recalcColumns(sData)
self.rawAddColumns(sData)
if self.opts.addTimes:
data.addTimes(time=self.opts.time,
times=sData.data[self.opts.time],
interpolate=self.opts.interpolateNewTime)
for n in sData.names():
if n != self.opts.time and (self.opts.columns == []
or data.validName(
n, self.opts.columns, True)):
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate)
data.append(diffs[i + 1] + " " + n, d)
self.joinedAddColumns(data)
data.rename(self.processName, renameTime=True)
data.rename(lambda c: c.strip())
data.eliminatedNames = None
if len(sources) > 1:
self.printColumns("written data", data)
if self.opts.automaticFormat:
if self.getDataFormat(dest) == "excel":
self.opts.writeExcel = True
if self.opts.writeExcel:
from pandas import ExcelWriter
with ExcelWriter(dest) as writer:
data.getData().to_excel(writer, sheet_name="Data")
if self.opts.addSheets:
for n, d in enumerate(rawData):
d.getData().to_excel(writer,
sheet_name="Original file %d" % n)
if hasXlsxWriter:
if len(self.opts.addFormulas) > 0:
from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell
rows = len(data.getData())
sheet = writer.sheets["Data"]
cols = {}
for i, n in enumerate(data.names()):
cols[n] = i
newC = i
for f in self.opts.addFormulas:
newC += 1
name, formula = f.split(":::")
sheet.write(0, newC, name)
cols[name] = newC
splitted = []
ind = 0
while ind >= 0:
if ind >= len(formula):
break
nInd = formula.find("'", ind)
if nInd < 0:
splitted.append(formula[ind:])
ind = nInd
elif nInd != ind:
splitted.append(formula[ind:nInd])
ind = nInd
else:
nInd = formula.find("'", ind + 1)
if nInd < 0:
self.error("No closing ' in formula",
formula)
name = formula[ind + 1:nInd]
if name not in cols:
self.error("Name", name,
"not in column names",
cols.keys())
splitted.append(cols[name])
ind = nInd + 1
for row in range(rows):
cellFormula = "="
for s in splitted:
if type(s) == int:
cellFormula += rowCol2Cell(row + 1, s)
else:
cellFormula += s
sheet.write(row + 1, newC, cellFormula)
print_(
"Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9"
)
else:
data.writeCSV(dest, delimiter=self.opts.delimiter)