def cleanUp(configParameters, currentTime, daysOfPicklesToKeep): """ @summary: Based on current time and frequencies contained within the time parameters, we will run the cleaners that need to be run. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner") if updateManager.updateIsRequired(currentTime): output = commands.getoutput(paths.STATSTOOLS + "pickleCleaner.py %s" % int(daysOfPicklesToKeep)) print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep) updateManager.addAutomaticUpdateToLogs(currentTime) updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner") if updateManager.updateIsRequired(currentTime): commands.getstatusoutput(paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf") print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" updateManager.addAutomaticUpdateToLogs(currentTime)
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics): """ @summary : Generates the weekly graphics that were not generated between last update and timeOfRequest @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingWeek in missingWeeks: self.timeOfRequest = missingWeek self.__generateAllRRDGraphicsForWebPage("weekly", generateTotalsGraphics) self.__generateAllGraphicsForGroups("weekly") self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays: self.timeOfRequest = missingDay self.__generateAllGraphicsForDailyWebPage(False, True) self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : Generates the monthly graphics that were not generated between last update and timeOfRequest """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingYear in missingYears: self.timeOfRequest = missingYear self.__generateAllRRDGraphicsForWebPage("yearly", True) self.__generateAllGraphicsForGroups("yearly") self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) missingDays.append(self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays[1:]: self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay) self.__generateAllForDailyWebPage(False, generateTotalsGraphics) self.__generateAllGraphicsForGroups("daily") self.timeOfRequest = oldTimeOfRequest
def backupRRDDatabases(configParameters, currentTime, nbBackupsToKeep): """ @summary: Based on current time and frequencies contained within the time parameters, we will backup the databases only if necessary. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "dbBackups") if updateManager.updateIsRequired(currentTime): commands.getstatusoutput(paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str(int(nbBackupsToKeep))) print paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str( nbBackupsToKeep) updateManager.addAutomaticUpdateToLogs(currentTime)
def cleanUp( configParameters, currentTime, daysOfPicklesToKeep ): """ @summary: Based on current time and frequencies contained within the time parameters, we will run the cleaners that need to be run. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner") if updateManager.updateIsRequired(currentTime) : output = commands.getoutput( paths.STATSTOOLS + "pickleCleaner.py %s" %int(daysOfPicklesToKeep) ) print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep ) updateManager.addAutomaticUpdateToLogs( currentTime ) updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner") if updateManager.updateIsRequired(currentTime) : commands.getstatusoutput( paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" ) print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" updateManager.addAutomaticUpdateToLogs( currentTime )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self): """ @summary : Generates all the csv files required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ #costs yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS monthlyCosts = yearlyCosts / 12.0 weeklyCosts = yearlyCosts / 52.0 #Get params from configuration files configParameters = StatsConfigParameters() configParameters.getAllParameters() clusters = str(configParameters.sourceMachinesTags).replace( '[', '').replace(']', '').replace(' ', '').replace('"', '').replace("'", "") updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForDailyWebPage(True, clusters, 0) if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts) if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts) if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def monitorActivities( configParameters, currentTime ): """ @summary: Monitors all the activities that occured during the course of this program. Report is sent out by mail to recipients specified in the config file. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "monitoring") if updateManager.updateIsRequired(currentTime) : output = commands.getoutput( paths.STATSBIN + "statsMonitor.py '%s'" %( currentTime ) ) print paths.STATSBIN + "statsMonitor.py" print output updateManager.addAutomaticUpdateToLogs( currentTime )
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : Generates the monthly graphics that were not generated between last update and timeOfRequest """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingYear in missingYears: self.timeOfRequest = missingYear self.__generateAllRRDGraphicsForWebPage( "yearly", True ) self.__generateAllGraphicsForGroups( "yearly" ) self.timeOfRequest = oldTimeOfRequest
def backupRRDDatabases( configParameters, currentTime, nbBackupsToKeep ): """ @summary: Based on current time and frequencies contained within the time parameters, we will backup the databases only if necessary. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "dbBackups" ) if updateManager.updateIsRequired( currentTime ) : commands.getstatusoutput( paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str( int(nbBackupsToKeep)) ) print paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str(nbBackupsToKeep) updateManager.addAutomaticUpdateToLogs( currentTime )
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) missingDays.append(self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays[1:]: self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay ) self.__generateAllForDailyWebPage( False, generateTotalsGraphics ) self.__generateAllGraphicsForGroups( "daily" ) self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ): """ @summary : Generates the weekly graphics that were not generated between last update and timeOfRequest @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingWeek in missingWeeks: self.timeOfRequest = missingWeek self.__generateAllRRDGraphicsForWebPage( "weekly", generateTotalsGraphics ) self.__generateAllGraphicsForGroups( "weekly" ) self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost ): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays: self.timeOfRequest = missingDay self.__generateAllGraphicsForDailyWebPage( False, True ) self.timeOfRequest = oldTimeOfRequest
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self): """ @summary : Gets all the graphics required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } #-------------------- print "time of the request : ", self.timeOfRequest # print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForDailyWebPage(True, True, True) # print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency]( self.timeOfRequest) == True: #print "weeklies need to be updated." self.generateAllForWeeklyWebPage(True, True) # print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForMonthlyWebPage(True, True) # print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForYearlyWebPage(True, True)
def monitorActivities(configParameters, currentTime): """ @summary: Monitors all the activities that occured during the course of this program. Report is sent out by mail to recipients specified in the config file. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "monitoring") if updateManager.updateIsRequired(currentTime): output = commands.getoutput(paths.STATSBIN + "statsMonitor.py '%s'" % (currentTime)) print paths.STATSBIN + "statsMonitor.py" print output updateManager.addAutomaticUpdateToLogs(currentTime)
def main(): """ @summary : Gets all the parameters from config file. Updates pickle files. Generates all the required graphics. Generates therwuired csv files. Updates the different web pages. Updates the desired databases. Uploads graphics to the required machines. Monitors the result of all the activities. """ if GeneralStatsLibraryMethods.processIsAlreadyRunning( "pxStatsStartup") == False: setGlobalLanguageParameters() GeneralStatsLibraryMethods.createLockFile("pxStatsStartup") currentTime = time.time() currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch(currentTime) generalParameters = StatsConfigParameters() generalParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() validateParameters(generalParameters, machineParameters, None) tagsNeedingUpdates = getMachinesTagsNeedingUpdates( generalParameters, machineParameters) if tagsNeedingUpdates == None: #no previous parameter found saveCurrentMachineParameters(machineParameters) elif tagsNeedingUpdates != []: updateFilesAssociatedWithMachineTags(tagsNeedingUpdates, machineParameters) saveCurrentMachineParameters(machineParameters) updatePickles(generalParameters, machineParameters, currentTimeInIsoFormat) updateDatabases(generalParameters, machineParameters, currentTimeInIsoFormat) backupRRDDatabases(generalParameters, currentTimeInIsoFormat, generalParameters.nbDbBackupsToKeep) #updateCsvFiles( ) getGraphicsForWebPages(generalParameters, currentTimeInIsoFormat) updateWebPages(generalParameters) #uploadGraphicFiles( generalParameters, machineParameters ) cleanUp(generalParameters, currentTimeInIsoFormat, generalParameters.daysOfPicklesToKeep) monitorActivities(generalParameters, currentTimeInIsoFormat) updateManager = AutomaticUpdatesManager( generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") updateManager.addAutomaticUpdateToLogs(currentTimeInIsoFormat) GeneralStatsLibraryMethods.deleteLockFile("pxStatsStartup") print _("Finished.") else: print _( "Error. An other instance of pxStatsStartup is allready running.") print _("Only one instance of this software can be run at once.") print _( "Please terminate the other instance or wait for it to end it's execution" ) print _("before running this program again.") print _("Program terminated.") sys.exit()
def main(): """ @summary : Gets all the parameters from config file. Updates pickle files. Generates all the required graphics. Generates therwuired csv files. Updates the different web pages. Updates the desired databases. Uploads graphics to the required machines. Monitors the result of all the activities. """ if GeneralStatsLibraryMethods.processIsAlreadyRunning( "pxStatsStartup" ) == False: setGlobalLanguageParameters() GeneralStatsLibraryMethods.createLockFile( "pxStatsStartup" ) currentTime = time.time() currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch( currentTime ) generalParameters = StatsConfigParameters() generalParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() validateParameters( generalParameters, machineParameters, None ) tagsNeedingUpdates = getMachinesTagsNeedingUpdates( generalParameters, machineParameters ) if tagsNeedingUpdates == None : #no previous parameter found saveCurrentMachineParameters( machineParameters ) elif tagsNeedingUpdates != [] : updateFilesAssociatedWithMachineTags( tagsNeedingUpdates, machineParameters ) saveCurrentMachineParameters( machineParameters ) updatePickles( generalParameters, machineParameters, currentTimeInIsoFormat ) updateDatabases( generalParameters, machineParameters, currentTimeInIsoFormat ) backupRRDDatabases( generalParameters, currentTimeInIsoFormat, generalParameters.nbDbBackupsToKeep ) #updateCsvFiles( ) getGraphicsForWebPages( generalParameters, currentTimeInIsoFormat ) updateWebPages( generalParameters ) #uploadGraphicFiles( generalParameters, machineParameters ) cleanUp( generalParameters , currentTimeInIsoFormat, generalParameters.daysOfPicklesToKeep ) monitorActivities( generalParameters, currentTimeInIsoFormat ) updateManager = AutomaticUpdatesManager( generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) updateManager.addAutomaticUpdateToLogs( currentTimeInIsoFormat ) GeneralStatsLibraryMethods.deleteLockFile( "pxStatsStartup" ) print _( "Finished." ) else: print _( "Error. An other instance of pxStatsStartup is allready running." ) print _( "Only one instance of this software can be run at once." ) print _( "Please terminate the other instance or wait for it to end it's execution" ) print _( "before running this program again." ) print _( "Program terminated." ) sys.exit()