def getSelectDataFromLog(startPos, stopPos, fields):
while (startPos != stopPos):
nextName = None
try:
f = open(startPos[0], 'r')
except IOError:
f = gzip.open(startPos[0] + ".gz", 'r')
f.seek(startPos[1])
lines = []
for i in range(const.numLines):
line = f.readline()
if line != "":
lines.append(line)
else:
nextName = getNextFilename(startPos[0])
break
startPos = (startPos[0], f.tell())
f.close()
if nextName:
startPos = (nextName, 0)
if len(lines) != 0:
writeData(startPos, pData.processData(lines), fields)
else:
writeData(startPos, None, fields)
print startPos, stopPos
def getDataFromLog(fname, tell):
"""Get some data starting with the given file name and tell location."""
nextName = None
try:
f = open(fname, 'r')
except IOError:
f = gzip.open(fname + ".gz", 'r')
f.seek(tell)
lines = []
for i in range(const.numLines):
line = f.readline()
if line != "":
lines.append(line)
else:
nextName = getNextFilename(fname)
break
position = f.tell()
f.close()
if nextName:
fname = nextName
position = 0
if len(lines) != 0:
writeData((fname, position), pData.processData(lines), models.addToEntry.keys())
else:
writeData((fname, position), None, models.addToEntry.keys())
return fname, position
def logDataCheck(startPos):
"""Need to modify this to act inside of a loop. Will get to that later."""
with open(logName, 'r') as f:
f.seek(startPos)
nextLines = list(islice(f, const.numLines))
if not nextLines:
sys.exit()
print nextLines
startPos = f.tell()
writeData(startPos, pData.processData(nextLines))
