def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] data = SpreadsheetData(csvName=sources[0], title=path.splitext(path.basename(sources[0]))[0]) if self.opts.time == None: self.opts.time = data.names()[0] for s in sources[1:]: addition = path.splitext(path.basename(s))[0] sData = SpreadsheetData(csvName=s) if self.opts.addTimes: data.addTimes( time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime ) for n in sData.names(): if n != self.opts.time: d = data.resample( sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate, ) data.append(addition + " " + n, d, allowDuplicates=True) data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] data = SpreadsheetData(txtName=sources[0], timeName=self.opts.time, validData=self.opts.columns, title=path.splitext(path.basename( sources[0]))[0]) if self.opts.time == None: self.opts.time = data.names()[0] for s in sources[1:]: addition = path.splitext(path.basename(s))[0] sData = SpreadsheetData(txtName=s) for n in sData.names(): if n != self.opts.time and (self.opts.columns == [] or n in self.opts.columns): d = data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(addition + " " + n, d) data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=self.parser.getArgs()[0:-1] data=SpreadsheetData(txtName=sources[0], timeName=self.opts.time, validData=self.opts.columns, title=path.splitext(path.basename(sources[0]))[0]) if self.opts.time==None: self.opts.time=data.names()[0] for s in sources[1:]: addition=path.splitext(path.basename(s))[0] sData=SpreadsheetData(txtName=s) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or n in self.opts.columns): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(addition+" "+n,d) data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] data = SpreadsheetData(csvName=sources[0], title=path.splitext(path.basename( sources[0]))[0]) if self.opts.time == None: self.opts.time = data.names()[0] for s in sources[1:]: addition = path.splitext(path.basename(s))[0] sData = SpreadsheetData(csvName=s) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n != self.opts.time: d = data.resample( sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(addition + " " + n, d, allowDuplicates=True) data.writeCSV(dest, delimiter=self.opts.delimiter)
def testSpreadsheetReadFileTest(self): fName="/tmp/testdata" open(fName,"w").write(filecontent) sp=SpreadsheetData(txtName=fName) self.assertEqual(len(sp.names()),4) self.assertEqual(sp.size(),5) self.assertEqual(sp.tRange(),(0.005,0.025)) self.assertEqual(sp.names()[1],"Initial")
def testSpreadsheetReadFileTest(self): fName = mktemp() open(fName, "wb").write(filecontent) sp = SpreadsheetData(txtName=fName) self.assertEqual(len(sp.names()), 4) self.assertEqual(sp.size(), 5) self.assertEqual(sp.tRange(), (0.005, 0.025)) self.assertEqual(sp.names()[1], "Initial")
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] data = SpreadsheetData(title=path.splitext(path.basename( sources[0]))[0], validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0], data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time == None: self.opts.time = data.names()[0] for s in sources[1:]: addition = path.splitext(path.basename(s))[0] sData = SpreadsheetData(validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, **self.dataFormatOptions(s)) self.printColumns(s, sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n != self.opts.time: d = data.resample( sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(addition + " " + n, d, allowDuplicates=True) self.joinedAddColumns(data) if len(sources) > 1: self.printColumns("written data", data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=self.parser.getArgs()[0:-1] data=SpreadsheetData(title=path.splitext(path.basename(sources[0]))[0], validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.names()[0] for s in sources[1:]: addition=path.splitext(path.basename(s))[0] sData=SpreadsheetData(validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, **self.dataFormatOptions(s)) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n!=self.opts.time: d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(addition+" "+n, d, allowDuplicates=True) self.joinedAddColumns(data) if len(sources)>1: self.printColumns("written data",data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def testGetSpreadsheet(self): csvName="/tmp/nix.csv" sd=TimelineDirectory(theDir) st=sd["p"] spread=st() spread.writeCSV(csvName) self.assertEqual(len(st.positions)+1,len(spread.names())) rereadSpread=SpreadsheetData(csvName=csvName) self.assertEqual(len(spread.names()),len(rereadSpread.names())) self.assertEqual(len(spread.data),len(rereadSpread.data)) sd=TimelineDirectory(theDir,"timeline") st=sd["p"] spread=st() spread.writeCSV("/tmp/nix2.csv")
def testGetSpreadsheet(self): csvName = path.join(self.theDir, "nix.csv") sd = TimelineDirectory(self.theDir) st = sd["p"] spread = st() spread.writeCSV(csvName) self.assertEqual(len(st.positions) + 1, len(spread.names())) rereadSpread = SpreadsheetData(csvName=csvName) self.assertEqual(len(spread.names()), len(rereadSpread.names())) self.assertEqual(len(spread.data), len(rereadSpread.data)) sd = TimelineDirectory(self.theDir, "timeline") st = sd["p"] spread = st() spread.writeCSV(path.join(self.theDir, "nix2.csv"))
def testGetSpreadsheet(self): csvName = path.join(self.theDir, "nix.csv") sd = TimelineDirectory(self.theDir) st = sd["p"] spread = st() assert len(spread.names()) == 5 assert spread.size() == 3 spread.writeCSV(csvName) assert len(st.positions) + 1 == len(spread.names()) rereadSpread = SpreadsheetData(csvName=csvName) assert len(spread.names()) == len(rereadSpread.names()) assert len(spread.data) == len(rereadSpread.data) sd = TimelineDirectory(self.theDir, "timeline") st = sd["p"] spread = st() spread.writeCSV(path.join(self.theDir, "nix2.csv"))
def testGetSpreadsheet(self): csvName=path.join(self.theDir,"nix.csv") sd=TimelineDirectory(self.theDir) st=sd["p"] spread=st() assert len(spread.names())==5 assert spread.size()==3 spread.writeCSV(csvName) assert len(st.positions)+1==len(spread.names()) rereadSpread=SpreadsheetData(csvName=csvName) assert len(spread.names())==len(rereadSpread.names()) assert len(spread.data)==len(rereadSpread.data) sd=TimelineDirectory(self.theDir,"timeline") st=sd["p"] spread=st() spread.writeCSV(path.join(self.theDir,"nix2.csv"))
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=self.parser.getArgs()[0:-1] diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=None title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.names()[0] for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) if len(sources)>1: self.printColumns("written data",data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def testSpreadsheetDataExtend(self): sp1 = SpreadsheetData(data=data1, names=names1) sp1.append("test", [i * i for i in range(len(data1))]) self.assertEqual(len(sp1.names()), len(names1) + 1) self.assert_("test" in sp1.names())
def testSpreadsheetReadFileHandleTest(self): sp = SpreadsheetData(txtName=BytesIO(filecontent)) self.assertEqual(len(sp.names()), 4)
def testSpreadsheetReadFileHandleTest(self): fName = mktemp() open(fName, "w").write(filecontent) sp = SpreadsheetData(txtName=open(fName)) self.assertEqual(len(sp.names()), 4)
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=[] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/")>=0: front,back=s.split("/*lastTime*/",1) for d in glob(front): lastTime=None for f in listdir(d): if path.exists(path.join(d,f,back)): try: t=float(f) if lastTime: if t>float(lastTime): lastTime=f else: lastTime=f except ValueError: pass if lastTime: sources.append(path.join(d,lastTime,back)) else: sources.append(s) diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=None title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData=[deepcopy(data)] self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.timeName() if not diffs[0] is None: data.rename(lambda c:diffs[0]+" "+c) for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) data.rename(self.processName,renameTime=True) data.rename(lambda c:c.strip()) if len(sources)>1: self.printColumns("written data",data) if self.opts.automaticFormat: if self.getDataFormat(dest)=="excel": self.opts.writeExcel=True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer) if self.opts.addSheets: for n,d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def testSpreadsheetDataConstruction(self): sp = SpreadsheetData(data=data1, names=names1) self.assertEqual(names1, list(sp.names())) self.assertEqual(len(sp.data), len(names1) * 2) self.assertEqual(max(sp.data['t'] - range(len(names1) * 2)), 0)
def testSpreadsheetDataConstruction(self): sp=SpreadsheetData(data=data1,names=names1) self.assertEqual(names1,list(sp.names())) self.assertEqual(len(sp.data),len(names1)*2) self.assertEqual(max(sp.data['t']-range(len(names1)*2)),0)
def testSpreadsheetReadFileHandleTest(self): fName="/tmp/testdata" open(fName,"w").write(filecontent) sp=SpreadsheetData(txtName=open(fName)) self.assertEqual(len(sp.names()),4)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] diffs = [None] if len(sources) > 1: # find differing parts commonStart = 1e4 commonEnd = 1e4 for s in sources[1:]: a = path.abspath(sources[0]) b = path.abspath(s) start = 0 end = 0 for i in range(min(len(a), len(b))): start = i if a[i] != b[i]: break commonStart = min(commonStart, start) for i in range(min(len(a), len(b))): end = i if a[-(i + 1)] != b[-(i + 1)]: break commonEnd = min(commonEnd, end) diffs = [] for s in sources: b = path.abspath(s) if commonEnd > 0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names = None title = path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names = path.splitext(path.basename(sources[0]))[0].split("_") title = None data = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0], data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time == None: self.opts.time = data.names()[0] for i, s in enumerate(sources[1:]): names = None title = path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names = title.split("_") title = None sData = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) self.printColumns(s, sData) self.recalcColumns(sData) self.rawAddColumns(sData) for n in sData.names(): if n != self.opts.time and (self.opts.columns == [] or data.validName( n, self.opts.columns, True)): d = data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(diffs[i + 1] + " " + n, d) self.joinedAddColumns(data) if len(sources) > 1: self.printColumns("written data", data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
class ReplayDataFileAnalyzer(GeneralLineAnalyzer): """Reads Data into a SpreadsheetData-object and then every time a new time value comes up returns the data from that file""" def __init__(self, timeName=None, validData=None, validMatchRegexp=False, csvName=None, txtName=None, excelName=None, namePrefix=None, skip_header=0, stripCharacters=None, progressTemplate=None, replaceFirstLine=None, startTime=None, endTime=None): self.__data=SpreadsheetData(timeName=timeName, validData=validData, validMatchRegexp=validMatchRegexp, csvName=csvName, txtName=txtName, excelName=excelName, skip_header=skip_header, stripCharacters=stripCharacters, replaceFirstLine=replaceFirstLine) self.namePrefix=namePrefix GeneralLineAnalyzer.__init__(self, titles=[self.makeName(n) for n in self.__data.names(withTime=False)], doTimelines=True, doFiles=False, progressTemplate=progressTemplate, startTime=startTime, endTime=endTime) self.timeNew=False self.tRange=self.__data.tRange() def makeName(self,name): if self.namePrefix is None: return name else: return self.namePrefix+name def timeChanged(self): time=float(self.getTime()) if time<self.tRange[0] or time>self.tRange[1]: return self.timeNew=True self.lines.setTime(time) def doMatch(self,line): if self.timeNew: time=float(self.getTime()) if time<self.tRange[0] or time>self.tRange[1]: return None data=self.__data(time,invalidExtend=True) self.timeNew=False return {self.makeName(k):v for k,v in data.items()} else: return None def addToTimelines(self,data): for t in self.titles: self.lines.setValue(self.fName(t),data[t]) def processProgressTemplate(self,data): ldata={k[len(self.namePrefix):]:data[k] for k in data} myProgress=self.progressTemplate.format(**ldata) return myProgress
def testSpreadsheetReadFileHandleTest(self): sp=SpreadsheetData(txtName=StringIO(filecontent)) self.assertEqual(len(sp.names()),4)
def testSpreadsheetDataExtend(self): sp1=SpreadsheetData(data=data1,names=names1) sp1.append("test",[i*i for i in range(len(data1))]) self.assertEqual(len(sp1.names()),len(names1)+1) self.assert_("test" in sp1.names())
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=[] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/")>=0: front,back=s.split("/*lastTime*/",1) for d in glob(front): lastTime=None for f in listdir(d): if path.exists(path.join(d,f,back)): try: t=float(f) if lastTime: if t>float(lastTime): lastTime=f else: lastTime=f except ValueError: pass if lastTime: sources.append(path.join(d,lastTime,back)) else: sources.append(s) diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=self.names title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: if not names is None: self.error("Names already specified as",names,". Can't calc from filename") names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData=[deepcopy(data)] self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.timeName() if not diffs[0] is None: data.rename(lambda c:diffs[0]+" "+c) for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) data.rename(self.processName,renameTime=True) data.rename(lambda c:c.strip()) data.eliminatedNames=None if len(sources)>1: self.printColumns("written data",data) if self.opts.automaticFormat: if self.getDataFormat(dest)=="excel": self.opts.writeExcel=True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer,sheet_name="Data") if self.opts.addSheets: for n,d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) if hasXlsxWriter: if len(self.opts.addFormulas)>0: from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell rows=len(data.getData()) sheet=writer.sheets["Data"] cols={} for i,n in enumerate(data.names()): cols[n]=i newC=i for f in self.opts.addFormulas: newC+=1 name,formula=f.split(":::") sheet.write(0,newC,name) cols[name]=newC splitted=[] ind=0 while ind>=0: if ind>=len(formula): break nInd=formula.find("'",ind) if nInd<0: splitted.append(formula[ind:]) ind=nInd elif nInd!=ind: splitted.append(formula[ind:nInd]) ind=nInd else: nInd=formula.find("'",ind+1) if nInd<0: self.error("No closing ' in formula",formula) name=formula[ind+1:nInd] if name not in cols: self.error("Name",name,"not in column names",cols.keys()) splitted.append(cols[name]) ind=nInd+1 for row in range(rows): cellFormula="=" for s in splitted: if type(s)==int: cellFormula+=rowCol2Cell(row+1,s) else: cellFormula+=s sheet.write(row+1,newC,cellFormula) print_("Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9") else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = [] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/") >= 0: front, back = s.split("/*lastTime*/", 1) for d in glob(front): lastTime = None for f in listdir(d): if path.exists(path.join(d, f, back)): try: t = float(f) if lastTime: if t > float(lastTime): lastTime = f else: lastTime = f except ValueError: pass if lastTime: sources.append(path.join(d, lastTime, back)) else: sources.append(s) diffs = [None] if len(sources) > 1: # find differing parts commonStart = 1e4 commonEnd = 1e4 for s in sources[1:]: a = path.abspath(sources[0]) b = path.abspath(s) start = 0 end = 0 for i in range(min(len(a), len(b))): start = i if a[i] != b[i]: break commonStart = min(commonStart, start) for i in range(min(len(a), len(b))): end = i if a[-(i + 1)] != b[-(i + 1)]: break commonEnd = min(commonEnd, end) diffs = [] for s in sources: b = path.abspath(s) if commonEnd > 0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names = self.names title = path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: if not names is None: self.error("Names already specified as", names, ". Can't calc from filename") names = path.splitext(path.basename(sources[0]))[0].split("_") title = None data = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData = [deepcopy(data)] self.printColumns(sources[0], data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time == None: self.opts.time = data.timeName() if not diffs[0] is None: data.rename(lambda c: diffs[0] + " " + c) for i, s in enumerate(sources[1:]): names = None title = path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names = title.split("_") title = None sData = SpreadsheetData( names=names, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s, sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n != self.opts.time and (self.opts.columns == [] or data.validName( n, self.opts.columns, True)): d = data.resample( sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i + 1] + " " + n, d) self.joinedAddColumns(data) data.rename(self.processName, renameTime=True) data.rename(lambda c: c.strip()) data.eliminatedNames = None if len(sources) > 1: self.printColumns("written data", data) if self.opts.automaticFormat: if self.getDataFormat(dest) == "excel": self.opts.writeExcel = True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer, sheet_name="Data") if self.opts.addSheets: for n, d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) if hasXlsxWriter: if len(self.opts.addFormulas) > 0: from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell rows = len(data.getData()) sheet = writer.sheets["Data"] cols = {} for i, n in enumerate(data.names()): cols[n] = i newC = i for f in self.opts.addFormulas: newC += 1 name, formula = f.split(":::") sheet.write(0, newC, name) cols[name] = newC splitted = [] ind = 0 while ind >= 0: if ind >= len(formula): break nInd = formula.find("'", ind) if nInd < 0: splitted.append(formula[ind:]) ind = nInd elif nInd != ind: splitted.append(formula[ind:nInd]) ind = nInd else: nInd = formula.find("'", ind + 1) if nInd < 0: self.error("No closing ' in formula", formula) name = formula[ind + 1:nInd] if name not in cols: self.error("Name", name, "not in column names", cols.keys()) splitted.append(cols[name]) ind = nInd + 1 for row in range(rows): cellFormula = "=" for s in splitted: if type(s) == int: cellFormula += rowCol2Cell(row + 1, s) else: cellFormula += s sheet.write(row + 1, newC, cellFormula) print_( "Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9" ) else: data.writeCSV(dest, delimiter=self.opts.delimiter)