def cleanUp(configParameters, currentTime, daysOfPicklesToKeep):
"""
@summary: Based on current time and frequencies contained
within the time parameters, we will run
the cleaners that need to be run.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner")
if updateManager.updateIsRequired(currentTime):
output = commands.getoutput(paths.STATSTOOLS + "pickleCleaner.py %s" %
int(daysOfPicklesToKeep))
print paths.STATSTOOLS + "pickleCleaner.py" + " " + str(
daysOfPicklesToKeep)
updateManager.addAutomaticUpdateToLogs(currentTime)
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner")
if updateManager.updateIsRequired(currentTime):
commands.getstatusoutput(paths.STATSTOOLS + "clean_dir.plx" + " " +
paths.STATSETC + "clean_dir.conf")
print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf"
updateManager.addAutomaticUpdateToLogs(currentTime)
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingWeeks = updateManager.getMissingWeeksBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage("weekly",
generateTotalsGraphics)
self.__generateAllGraphicsForGroups("weekly")
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage(False, True)
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingYears = updateManager.getMissingYearsBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage("yearly", True)
self.__generateAllGraphicsForGroups("yearly")
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay)
self.__generateAllForDailyWebPage(False, generateTotalsGraphics)
self.__generateAllGraphicsForGroups("daily")
self.timeOfRequest = oldTimeOfRequest
def backupRRDDatabases(configParameters, currentTime, nbBackupsToKeep):
"""
@summary: Based on current time and frequencies contained
within the time parameters, we will backup the databases
only if necessary.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "dbBackups")
if updateManager.updateIsRequired(currentTime):
commands.getstatusoutput(paths.STATSTOOLS + "backupRRDDatabases.py" +
" " + str(int(nbBackupsToKeep)))
print paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str(
nbBackupsToKeep)
updateManager.addAutomaticUpdateToLogs(currentTime)
def cleanUp( configParameters, currentTime, daysOfPicklesToKeep ):
"""
@summary: Based on current time and frequencies contained
within the time parameters, we will run
the cleaners that need to be run.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner")
if updateManager.updateIsRequired(currentTime) :
output = commands.getoutput( paths.STATSTOOLS + "pickleCleaner.py %s" %int(daysOfPicklesToKeep) )
print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep )
updateManager.addAutomaticUpdateToLogs( currentTime )
updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner")
if updateManager.updateIsRequired(currentTime) :
commands.getstatusoutput( paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" )
print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf"
updateManager.addAutomaticUpdateToLogs( currentTime )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Generates all the csv files required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
#costs
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
clusters = str(configParameters.sourceMachinesTags).replace(
'[', '').replace(']', '').replace(' ',
'').replace('"',
'').replace("'", "")
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, clusters, 0)
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def monitorActivities( configParameters, currentTime ):
"""
@summary: Monitors all the activities that occured during
the course of this program. Report is sent out by mail
to recipients specified in the config file.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "monitoring")
if updateManager.updateIsRequired(currentTime) :
output = commands.getoutput( paths.STATSBIN + "statsMonitor.py '%s'" %( currentTime ) )
print paths.STATSBIN + "statsMonitor.py"
print output
updateManager.addAutomaticUpdateToLogs( currentTime )
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage( "yearly", True )
self.__generateAllGraphicsForGroups( "yearly" )
self.timeOfRequest = oldTimeOfRequest
def backupRRDDatabases( configParameters, currentTime, nbBackupsToKeep ):
"""
@summary: Based on current time and frequencies contained
within the time parameters, we will backup the databases
only if necessary.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "dbBackups" )
if updateManager.updateIsRequired( currentTime ) :
commands.getstatusoutput( paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str( int(nbBackupsToKeep)) )
print paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str(nbBackupsToKeep)
updateManager.addAutomaticUpdateToLogs( currentTime )
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay )
self.__generateAllForDailyWebPage( False, generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "daily" )
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage( "weekly", generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "weekly" )
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage( False, True )
self.timeOfRequest = oldTimeOfRequest
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Gets all the graphics required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
#-------------------- print "time of the request : ", self.timeOfRequest
# print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, True, True)
# print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
#print "weeklies need to be updated."
self.generateAllForWeeklyWebPage(True, True)
# print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, True)
# print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, True)
def monitorActivities(configParameters, currentTime):
"""
@summary: Monitors all the activities that occured during
the course of this program. Report is sent out by mail
to recipients specified in the config file.
@param configParameters: StatsConfigParameters instance.
@param currenTime: currentTime in seconds since epoch format.
"""
paths = StatsPaths()
paths.setPaths()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "monitoring")
if updateManager.updateIsRequired(currentTime):
output = commands.getoutput(paths.STATSBIN + "statsMonitor.py '%s'" %
(currentTime))
print paths.STATSBIN + "statsMonitor.py"
print output
updateManager.addAutomaticUpdateToLogs(currentTime)
def main():
"""
@summary : Gets all the parameters from config file.
Updates pickle files.
Generates all the required graphics.
Generates therwuired csv files.
Updates the different web pages.
Updates the desired databases.
Uploads graphics to the required machines.
Monitors the result of all the activities.
"""
if GeneralStatsLibraryMethods.processIsAlreadyRunning(
"pxStatsStartup") == False:
setGlobalLanguageParameters()
GeneralStatsLibraryMethods.createLockFile("pxStatsStartup")
currentTime = time.time()
currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch(currentTime)
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
validateParameters(generalParameters, machineParameters, None)
tagsNeedingUpdates = getMachinesTagsNeedingUpdates(
generalParameters, machineParameters)
if tagsNeedingUpdates == None: #no previous parameter found
saveCurrentMachineParameters(machineParameters)
elif tagsNeedingUpdates != []:
updateFilesAssociatedWithMachineTags(tagsNeedingUpdates,
machineParameters)
saveCurrentMachineParameters(machineParameters)
updatePickles(generalParameters, machineParameters,
currentTimeInIsoFormat)
updateDatabases(generalParameters, machineParameters,
currentTimeInIsoFormat)
backupRRDDatabases(generalParameters, currentTimeInIsoFormat,
generalParameters.nbDbBackupsToKeep)
#updateCsvFiles( )
getGraphicsForWebPages(generalParameters, currentTimeInIsoFormat)
updateWebPages(generalParameters)
#uploadGraphicFiles( generalParameters, machineParameters )
cleanUp(generalParameters, currentTimeInIsoFormat,
generalParameters.daysOfPicklesToKeep)
monitorActivities(generalParameters, currentTimeInIsoFormat)
updateManager = AutomaticUpdatesManager(
generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
updateManager.addAutomaticUpdateToLogs(currentTimeInIsoFormat)
GeneralStatsLibraryMethods.deleteLockFile("pxStatsStartup")
print _("Finished.")
else:
print _(
"Error. An other instance of pxStatsStartup is allready running.")
print _("Only one instance of this software can be run at once.")
print _(
"Please terminate the other instance or wait for it to end it's execution"
)
print _("before running this program again.")
print _("Program terminated.")
sys.exit()
def main():
"""
@summary : Gets all the parameters from config file.
Updates pickle files.
Generates all the required graphics.
Generates therwuired csv files.
Updates the different web pages.
Updates the desired databases.
Uploads graphics to the required machines.
Monitors the result of all the activities.
"""
if GeneralStatsLibraryMethods.processIsAlreadyRunning( "pxStatsStartup" ) == False:
setGlobalLanguageParameters()
GeneralStatsLibraryMethods.createLockFile( "pxStatsStartup" )
currentTime = time.time()
currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch( currentTime )
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
validateParameters( generalParameters, machineParameters, None )
tagsNeedingUpdates = getMachinesTagsNeedingUpdates( generalParameters, machineParameters )
if tagsNeedingUpdates == None : #no previous parameter found
saveCurrentMachineParameters( machineParameters )
elif tagsNeedingUpdates != [] :
updateFilesAssociatedWithMachineTags( tagsNeedingUpdates, machineParameters )
saveCurrentMachineParameters( machineParameters )
updatePickles( generalParameters, machineParameters, currentTimeInIsoFormat )
updateDatabases( generalParameters, machineParameters, currentTimeInIsoFormat )
backupRRDDatabases( generalParameters, currentTimeInIsoFormat, generalParameters.nbDbBackupsToKeep )
#updateCsvFiles( )
getGraphicsForWebPages( generalParameters, currentTimeInIsoFormat )
updateWebPages( generalParameters )
#uploadGraphicFiles( generalParameters, machineParameters )
cleanUp( generalParameters , currentTimeInIsoFormat, generalParameters.daysOfPicklesToKeep )
monitorActivities( generalParameters, currentTimeInIsoFormat )
updateManager = AutomaticUpdatesManager( generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
updateManager.addAutomaticUpdateToLogs( currentTimeInIsoFormat )
GeneralStatsLibraryMethods.deleteLockFile( "pxStatsStartup" )
print _( "Finished." )
else:
print _( "Error. An other instance of pxStatsStartup is allready running." )
print _( "Only one instance of this software can be run at once." )
print _( "Please terminate the other instance or wait for it to end it's execution" )
print _( "before running this program again." )
print _( "Program terminated." )
sys.exit()