def _update_events(self):
earliestTime = get_current_gmtime()
curTime = time.time()
for logTypeName, remoteFolder in self.serverList.iteritems():
try:
log_msg("Updating logs from %s..." % (logTypeName))
dataDir = os.path.join(self.baseDataDir, logTypeName)
if not os.path.exists(dataDir):
os.makedirs(dataDir)
#get the changes from the remote server:
os.system('rsync --append -rtz -e "ssh -i %s" %s/ %s' % (IDENTITY, remoteFolder, dataDir))
#for each file in the folder
fileNames = glob.glob(dataDir + "/*")
for fileName in fileNames:
baseFileName = os.path.split(fileName)[1]
#ignore really old files:
if self._file_is_old(fileName, curTime):
self._remove_file(fileName, baseFileName, logTypeName)
continue
#look up the database row
results = self._get_file_row(baseFileName, logTypeName)
#if the row existed, figure out if it is old enough to be deleted
if len(results) > 0:
assert len(results) == 1, "Why are there two rows for %s and %s?" % (baseFileName, logTypeName)
numEvents, lastMTimeString = results[0]
rowExisted = True
if not self._file_was_modified(fileName, lastMTimeString):
#don't bother continuing to parse the file if it hasnt been modified
continue
#otherwise, just note that we've obviously never parsed any events from this file
else:
numEvents = 0
rowExisted = False
#load all lines
cur = self.conn.cursor()
try:
startTime, newNumEvents = EventLogging.parse_events(cur, fileName, numEvents)
if startTime < earliestTime:
earliestTime = startTime
log_msg("Parsed %s events" % (newNumEvents-numEvents))
#if any line fails, abort everything and log the failure
except Exception, error:
self._log_failure(error, "Failure (%s) while processing line from %s" % (error, fileName))
#otherwise update the file row in the database to note that we've successfully parsed newNumEvents events
else:
newMTimeString = str(os.path.getmtime(fileName))
if rowExisted:
self._update_file_row(cur, baseFileName, logTypeName, newNumEvents, newMTimeString)
else:
self._insert_file_row(cur, baseFileName, logTypeName, newNumEvents, newMTimeString)
finally:
cur.close()
self.conn.commit()
