def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=[] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/")>=0: front,back=s.split("/*lastTime*/",1) for d in glob(front): lastTime=None for f in listdir(d): if path.exists(path.join(d,f,back)): try: t=float(f) if lastTime: if t>float(lastTime): lastTime=f else: lastTime=f except ValueError: pass if lastTime: sources.append(path.join(d,lastTime,back)) else: sources.append(s) diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=self.names title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: if not names is None: self.error("Names already specified as",names,". Can't calc from filename") names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData=[deepcopy(data)] self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.timeName() if not diffs[0] is None: data.rename(lambda c:diffs[0]+" "+c) for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) data.rename(self.processName,renameTime=True) data.rename(lambda c:c.strip()) data.eliminatedNames=None if len(sources)>1: self.printColumns("written data",data) if self.opts.automaticFormat: if self.getDataFormat(dest)=="excel": self.opts.writeExcel=True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer,sheet_name="Data") if self.opts.addSheets: for n,d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) if hasXlsxWriter: if len(self.opts.addFormulas)>0: from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell rows=len(data.getData()) sheet=writer.sheets["Data"] cols={} for i,n in enumerate(data.names()): cols[n]=i newC=i for f in self.opts.addFormulas: newC+=1 name,formula=f.split(":::") sheet.write(0,newC,name) cols[name]=newC splitted=[] ind=0 while ind>=0: if ind>=len(formula): break nInd=formula.find("'",ind) if nInd<0: splitted.append(formula[ind:]) ind=nInd elif nInd!=ind: splitted.append(formula[ind:nInd]) ind=nInd else: nInd=formula.find("'",ind+1) if nInd<0: self.error("No closing ' in formula",formula) name=formula[ind+1:nInd] if name not in cols: self.error("Name",name,"not in column names",cols.keys()) splitted.append(cols[name]) ind=nInd+1 for row in range(rows): cellFormula="=" for s in splitted: if type(s)==int: cellFormula+=rowCol2Cell(row+1,s) else: cellFormula+=s sheet.write(row+1,newC,cellFormula) print_("Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9") else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=[] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/")>=0: front,back=s.split("/*lastTime*/",1) for d in glob(front): lastTime=None for f in listdir(d): if path.exists(path.join(d,f,back)): try: t=float(f) if lastTime: if t>float(lastTime): lastTime=f else: lastTime=f except ValueError: pass if lastTime: sources.append(path.join(d,lastTime,back)) else: sources.append(s) diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=None title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData=[deepcopy(data)] self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.timeName() if not diffs[0] is None: data.rename(lambda c:diffs[0]+" "+c) for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) data.rename(self.processName,renameTime=True) data.rename(lambda c:c.strip()) if len(sources)>1: self.printColumns("written data",data) if self.opts.automaticFormat: if self.getDataFormat(dest)=="excel": self.opts.writeExcel=True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer) if self.opts.addSheets: for n,d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest=self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file",dest,"exists already. Use --force to overwrite") sources=self.parser.getArgs()[0:-1] diffs=[None] if len(sources)>1: # find differing parts commonStart=1e4 commonEnd=1e4 for s in sources[1:]: a=path.abspath(sources[0]) b=path.abspath(s) start=0 end=0 for i in range(min(len(a),len(b))): start=i if a[i]!=b[i]: break commonStart=min(commonStart,start) for i in range(min(len(a),len(b))): end=i if a[-(i+1)]!=b[-(i+1)]: break commonEnd=min(commonEnd,end) diffs=[] for s in sources: b=path.abspath(s) if commonEnd>0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names=None title=path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names=path.splitext(path.basename(sources[0]))[0].split("_") title=None data=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0],data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time==None: self.opts.time=data.names()[0] for i,s in enumerate(sources[1:]): names=None title=path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names=title.split("_") title=None sData=SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) self.printColumns(s,sData) self.recalcColumns(sData) self.rawAddColumns(sData) for n in sData.names(): if n!=self.opts.time and (self.opts.columns==[] or data.validName(n,self.opts.columns,True)): d=data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(diffs[i+1]+" "+n,d) self.joinedAddColumns(data) if len(sources)>1: self.printColumns("written data",data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = self.parser.getArgs()[0:-1] diffs = [None] if len(sources) > 1: # find differing parts commonStart = 1e4 commonEnd = 1e4 for s in sources[1:]: a = path.abspath(sources[0]) b = path.abspath(s) start = 0 end = 0 for i in range(min(len(a), len(b))): start = i if a[i] != b[i]: break commonStart = min(commonStart, start) for i in range(min(len(a), len(b))): end = i if a[-(i + 1)] != b[-(i + 1)]: break commonEnd = min(commonEnd, end) diffs = [] for s in sources: b = path.abspath(s) if commonEnd > 0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names = None title = path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: names = path.splitext(path.basename(sources[0]))[0].split("_") title = None data = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) self.printColumns(sources[0], data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time == None: self.opts.time = data.names()[0] for i, s in enumerate(sources[1:]): names = None title = path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names = title.split("_") title = None sData = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) self.printColumns(s, sData) self.recalcColumns(sData) self.rawAddColumns(sData) for n in sData.names(): if n != self.opts.time and (self.opts.columns == [] or data.validName( n, self.opts.columns, True)): d = data.resample(sData, n, time=self.opts.time, extendData=self.opts.extendData) data.append(diffs[i + 1] + " " + n, d) self.joinedAddColumns(data) if len(sources) > 1: self.printColumns("written data", data) if self.opts.writeExcel: data.getData().to_excel(dest) else: data.writeCSV(dest, delimiter=self.opts.delimiter)
def run(self): dest = self.parser.getArgs()[-1] if path.exists(dest) and not self.opts.force: self.error("CSV-file", dest, "exists already. Use --force to overwrite") sources = [] for s in self.parser.getArgs()[0:-1]: if s.find("/*lastTime*/") >= 0: front, back = s.split("/*lastTime*/", 1) for d in glob(front): lastTime = None for f in listdir(d): if path.exists(path.join(d, f, back)): try: t = float(f) if lastTime: if t > float(lastTime): lastTime = f else: lastTime = f except ValueError: pass if lastTime: sources.append(path.join(d, lastTime, back)) else: sources.append(s) diffs = [None] if len(sources) > 1: # find differing parts commonStart = 1e4 commonEnd = 1e4 for s in sources[1:]: a = path.abspath(sources[0]) b = path.abspath(s) start = 0 end = 0 for i in range(min(len(a), len(b))): start = i if a[i] != b[i]: break commonStart = min(commonStart, start) for i in range(min(len(a), len(b))): end = i if a[-(i + 1)] != b[-(i + 1)]: break commonEnd = min(commonEnd, end) diffs = [] for s in sources: b = path.abspath(s) if commonEnd > 0: diffs.append(b[commonStart:-(commonEnd)]) else: diffs.append(b[commonStart:]) names = self.names title = path.splitext(path.basename(sources[0]))[0] if self.opts.namesFromFilename: if not names is None: self.error("Names already specified as", names, ". Can't calc from filename") names = path.splitext(path.basename(sources[0]))[0].split("_") title = None data = SpreadsheetData(names=names, timeName=self.opts.time, validData=self.opts.columns, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(sources[0])) rawData = [deepcopy(data)] self.printColumns(sources[0], data) self.recalcColumns(data) self.rawAddColumns(data) if self.opts.time == None: self.opts.time = data.timeName() if not diffs[0] is None: data.rename(lambda c: diffs[0] + " " + c) for i, s in enumerate(sources[1:]): names = None title = path.splitext(path.basename(s))[0] if self.opts.namesFromFilename: names = title.split("_") title = None sData = SpreadsheetData( names=names, skip_header=self.opts.skipHeaderLines, stripCharacters=self.opts.stripCharacters, replaceFirstLine=self.opts.replaceFirstLine, timeName=self.opts.time, validData=self.opts.columns, validMatchRegexp=self.opts.columnsRegexp, title=title, **self.dataFormatOptions(s)) rawData.append(sData) self.printColumns(s, sData) self.recalcColumns(sData) self.rawAddColumns(sData) if self.opts.addTimes: data.addTimes(time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime) for n in sData.names(): if n != self.opts.time and (self.opts.columns == [] or data.validName( n, self.opts.columns, True)): d = data.resample( sData, n, time=self.opts.time, extendData=self.opts.extendData, noInterpolation=not self.opts.newDataInterpolate) data.append(diffs[i + 1] + " " + n, d) self.joinedAddColumns(data) data.rename(self.processName, renameTime=True) data.rename(lambda c: c.strip()) data.eliminatedNames = None if len(sources) > 1: self.printColumns("written data", data) if self.opts.automaticFormat: if self.getDataFormat(dest) == "excel": self.opts.writeExcel = True if self.opts.writeExcel: from pandas import ExcelWriter with ExcelWriter(dest) as writer: data.getData().to_excel(writer, sheet_name="Data") if self.opts.addSheets: for n, d in enumerate(rawData): d.getData().to_excel(writer, sheet_name="Original file %d" % n) if hasXlsxWriter: if len(self.opts.addFormulas) > 0: from xlsxwriter.utility import xl_rowcol_to_cell as rowCol2Cell rows = len(data.getData()) sheet = writer.sheets["Data"] cols = {} for i, n in enumerate(data.names()): cols[n] = i newC = i for f in self.opts.addFormulas: newC += 1 name, formula = f.split(":::") sheet.write(0, newC, name) cols[name] = newC splitted = [] ind = 0 while ind >= 0: if ind >= len(formula): break nInd = formula.find("'", ind) if nInd < 0: splitted.append(formula[ind:]) ind = nInd elif nInd != ind: splitted.append(formula[ind:nInd]) ind = nInd else: nInd = formula.find("'", ind + 1) if nInd < 0: self.error("No closing ' in formula", formula) name = formula[ind + 1:nInd] if name not in cols: self.error("Name", name, "not in column names", cols.keys()) splitted.append(cols[name]) ind = nInd + 1 for row in range(rows): cellFormula = "=" for s in splitted: if type(s) == int: cellFormula += rowCol2Cell(row + 1, s) else: cellFormula += s sheet.write(row + 1, newC, cellFormula) print_( "Formulas written. In LibreOffice recalculate with Ctrl+Shift+F9" ) else: data.writeCSV(dest, delimiter=self.opts.delimiter)