def __generateAllMissingDailyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay)
self.__generateAllForDailyWebPage(False, generateTotalsGraphics)
self.__generateAllGraphicsForGroups("daily")
self.timeOfRequest = oldTimeOfRequest
def getTimeOfLastUpdateInLogs(self):
"""
@summary : Returns the time of the last update in iso format.
@return : None if no update as found, EPCH is returned in iso format,
as to make sure an update is made since no prior updates exist.
"""
timeOfLastUpdate = StatsDateLib.getIsoTodaysMidnight( StatsDateLib.getCurrentTimeInIsoformat() )
paths = StatsPaths()
paths.setPaths()
updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/"
if not os.path.isdir( updatesDirectory ):
os.makedirs(updatesDirectory)
allEntries = os.listdir(updatesDirectory)
if allEntries !=[] :
allEntries.sort()
allEntries.reverse()
timeOfLastUpdate = os.path.basename( allEntries[0] ).replace( "_"," " )
return timeOfLastUpdate
def getPreviousMonitoringJob( self, currentTime ):
"""
@summary : Gets the previous crontab from the pickle file.
@return : Time of the previous monitoring job.
@warning : Returns "" if file does not exist.
"""
statsPaths = StatsPaths()
statsPaths.setPaths()
file = "%spreviousMonitoringJob" %statsPaths.STATSMONITORING
previousMonitoringJob = ""
if os.path.isfile( file ):
fileHandle = open( file, "r" )
previousMonitoringJob = pickle.load( fileHandle )
fileHandle.close()
#print previousMonitoringJob
else:
previousMonitoringJob = StatsDateLib.getIsoTodaysMidnight( currentTime )
#print previousMonitoringJob
return previousMonitoringJob
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : start, end tuple in iso format.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
currentDate = datetime.date( int(currentTime[0:4]), int(currentTime[5:7]), 1 )
nbMonthsToRevwind = NB_MONTHS_DISPLAYED - 1
if currentDate.month - (nbMonthsToRevwind%12) < 1 :
month = currentDate.month - (nbMonthsToRevwind%12)+12
if currentDate.month -nbMonthsToRevwind < 1:
year = currentDate.year - int( abs(math.floor( float( ( currentDate.month - nbMonthsToRevwind ) / 12 ) ) ) )
else :
month = currentDate.month - nbMonthsToRevwind
year = currentDate.year
start = "%s-%s-%s 00:00:00" %( year,month,"01" )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def getStartEndOfWebPage():
"""
Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
"""
currentTime = StatsDateLib.getIsoFromEpoch(time.time())
start = StatsDateLib.rewindXDays(currentTime, (NB_YEARS_DISPLAYED - 1) * 365)
start = StatsDateLib.getIsoTodaysMidnight(start)
end = StatsDateLib.getIsoTodaysMidnight(currentTime)
return start, end
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : Start,end tuple both in ISO format.
"""
currentTime = StatsDateLib.getIsoFromEpoch(time.time())
start = StatsDateLib.rewindXDays(currentTime, NB_DAYS_DISPLAYED - 1)
start = StatsDateLib.getIsoTodaysMidnight(start)
end = StatsDateLib.getIsoTodaysMidnight(currentTime)
return start, end
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : Start,end tuple both in ISO format.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
start = StatsDateLib.rewindXDays( currentTime, NB_DAYS_DISPLAYED - 1 )
start = StatsDateLib.getIsoTodaysMidnight( start )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def getStartEndOfWebPage():
"""
Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
start = StatsDateLib.rewindXDays( currentTime, ( NB_YEARS_DISPLAYED - 1 ) * 365 )
start = StatsDateLib.getIsoTodaysMidnight( start )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay )
self.__generateAllForDailyWebPage( False, generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "daily" )
self.timeOfRequest = oldTimeOfRequest
def updateGroupedRoundRobinDatabases( infos, logger = None ):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch( infos.endTime )
tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients = infos.group, machines = infos.machines, fileType = infos.fileTypes[0] )
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0] )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( infos.endTime ) )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), infos.clients, infos.fileTypes[0], infos.machines )
#print timeSeperators
for i in xrange(0, len( timeSeperators ),2 ):#timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch( timeSeperators[i] )
dataPairs = getPairs( infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i+1], infos.group, logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = infos.group, groupName = infos.group, machines = infos.machines,fileType = infos.fileTypes[0], usage = "group" )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( rrdFileName, startTime, dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime and j < len( dataPairs[ dataType ] ):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0], endTime )
def updateRoundRobinDatabases( client, machines, fileType, endTime, logger = None ):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join( [machine for machine in machines ] )
tempRRDFileName = RrdUtilities.buildRRDFileName( dataType = _("errors"), clients = [client], machines = machines, fileType = fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, fileType )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( endTime ) )
endTime = StatsDateLib.getSecondsSinceEpoch( endTime )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), [client], fileType, machines )
for i in xrange( len(timeSeperators) -1 ) :
dataPairs = getPairs( [client], machines, fileType, timeSeperators[i], timeSeperators[i+1] , groupName = "", logger = logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = [client], machines = machines, fileType = fileType )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( databaseName = rrdFileName , startTime= startTime, dataType = dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime:
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
if logger != None :
try:
logger.info( _( "Updated %s db for %s in db named : %s" ) %( dataType, client, rrdFileName ) )
except:
pass
else:
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( rrdFileName, fileType, endTime )
def updateGroupedRoundRobinDatabases(infos, logger=None):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch(infos.endTime)
tempRRDFileName = RrdUtilities.buildRRDFileName(
_("errors"),
clients=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0])
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName,
infos.fileTypes[0])
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(infos.endTime))
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), infos.clients,
infos.fileTypes[0], infos.machines)
#print timeSeperators
for i in xrange(0, len(timeSeperators),
2): #timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch(timeSeperators[i])
dataPairs = getPairs(infos.clients, infos.machines, infos.fileTypes[0],
timeSeperators[i], timeSeperators[i + 1],
infos.group, logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=infos.group,
groupName=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0],
usage="group")
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(rrdFileName, startTime, dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime and j < len(
dataPairs[dataType]):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j + 1
for k in range(j, len(dataPairs[dataType])):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0],
endTime)
def updateRoundRobinDatabases(client,
machines,
fileType,
endTime,
logger=None):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join(
[machine for machine in machines])
tempRRDFileName = RrdUtilities.buildRRDFileName(dataType=_("errors"),
clients=[client],
machines=machines,
fileType=fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, fileType)
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(endTime))
endTime = StatsDateLib.getSecondsSinceEpoch(endTime)
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), [client], fileType, machines)
for i in xrange(len(timeSeperators) - 1):
dataPairs = getPairs([client],
machines,
fileType,
timeSeperators[i],
timeSeperators[i + 1],
groupName="",
logger=logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[client],
machines=machines,
fileType=fileType)
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(databaseName=rrdFileName,
startTime=startTime,
dataType=dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime:
j = j + 1
for k in range(j, len(dataPairs[dataType])):
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
if logger != None:
try:
logger.info(
_("Updated %s db for %s in db named : %s") %
(dataType, client, rrdFileName))
except:
pass
else:
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(rrdFileName, fileType,
endTime)
