def getSelectDataFromLog(startPos, stopPos, fields): while (startPos != stopPos): nextName = None try: f = open(startPos[0], 'r') except IOError: f = gzip.open(startPos[0] + ".gz", 'r') f.seek(startPos[1]) lines = [] for i in range(const.numLines): line = f.readline() if line != "": lines.append(line) else: nextName = getNextFilename(startPos[0]) break startPos = (startPos[0], f.tell()) f.close() if nextName: startPos = (nextName, 0) if len(lines) != 0: writeData(startPos, pData.processData(lines), fields) else: writeData(startPos, None, fields) print startPos, stopPos
def getDataFromLog(fname, tell): """Get some data starting with the given file name and tell location.""" nextName = None try: f = open(fname, 'r') except IOError: f = gzip.open(fname + ".gz", 'r') f.seek(tell) lines = [] for i in range(const.numLines): line = f.readline() if line != "": lines.append(line) else: nextName = getNextFilename(fname) break position = f.tell() f.close() if nextName: fname = nextName position = 0 if len(lines) != 0: writeData((fname, position), pData.processData(lines), models.addToEntry.keys()) else: writeData((fname, position), None, models.addToEntry.keys()) return fname, position
def logDataCheck(startPos): """Need to modify this to act inside of a loop. Will get to that later.""" with open(logName, 'r') as f: f.seek(startPos) nextLines = list(islice(f, const.numLines)) if not nextLines: sys.exit() print nextLines startPos = f.tell() writeData(startPos, pData.processData(nextLines))