def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays: self.timeOfRequest = missingDay self.__generateAllGraphicsForDailyWebPage(False, True) self.timeOfRequest = oldTimeOfRequest
def __generateAllGraphicsForGroups( self, graphicType ): """ @summary : Generated groups graphics based on the specified graphicType. @summary graphicType : "daily", "weekly", "monthly", "yearly" @raise Exception: When graphicType is unknown. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" } if graphicType not in supportedGraphicTypes: raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" ) else: for group in configParameters.groupParameters.groups: groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat( group ) groupMachines = str(groupMachines).replace( "[", "" ).replace( "]", "" ).replace( "'", "" ).replace( '"','' ) if graphicType == "daily": commands.getstatusoutput( '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage ) ) #print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage ) else: commands.getoutput("%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage ) ) print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage )
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : Generates the monthly graphics that were not generated between last update and timeOfRequest """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingYear in missingYears: self.timeOfRequest = missingYear self.__generateAllRRDGraphicsForWebPage("yearly", True) self.__generateAllGraphicsForGroups("yearly") self.timeOfRequest = oldTimeOfRequest
def calculateTotalsForEachColumn(lines, includeGroups=False): """ @summary : Goes through all the lines and makes the total of every field @param lines: Lines that we need to browse through. @return : list of totals for each fields. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups totals = [0.0 for i in range(len(lines[0].split(',')) - 1)] for i in range(len(lines[1:])): if lines[i + 1].split(',')[0].split(' ')[0] not in knownGroups: #print "original line : " + lines[i+1] values = lines[i + 1].split(',')[1:] #print "split up values : %s " %(values) for j in range(len(values)): totals[j] = totals[j] + float(values[j]) return totals
def getTranslatorForModule(moduleAbsPath, language=None): """ @summary : Returns a translator based the specified module and the language for which it is needed. @param moduleAbsPath: AbsolutePath name of the module for which we need the translation file. @param language: Language for whcih to find a proper translator. If none is specified, it will be set to the value found within the configuration file. @return: Return the translator to be used by _ . """ if language == None: configParameters = StatsConfigParameters() configParameters.getAllParameters() language = configParameters.mainApplicationLanguage fileName = LanguageTools.getTranslationFileName( language, moduleAbsPath) translator = LanguageTools.getTranslator(fileName) return translator
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile(): """ @summary : Reads the config file and returns all the currently running rx and tx names associated with any of the source machines found within the config file. @return: Returns the rxNames and the txNames found. """ rxNames = [] txNames = [] configParameters = StatsConfigParameters() configParameters.getAllParameters() for tag in configParameters.sourceMachinesTags: machine = configParameters.detailedParameters.sourceMachinesForTag[ tag][0] newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine) rxNames.extend(filter(lambda x: x not in rxNames, newRxNames)) txNames.extend(filter(lambda x: x not in txNames, newTxNames)) return rxNames, txNames
def __generateAllRRDGraphicsForWebPage(self, graphicType, generateTotalsGraphics=True): """ @summary : This method generates new rrd graphics based on the specified graphics @param graphicType : daily weekly monthly or yearly @raise Exception : When graphicType is unknown. """ supportedGraphicTypes = { "daily": "-d", "weekly": "-w", "monthly": "-m", "yearly": "-y" } if graphicType not in supportedGraphicTypes: raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" ) configParameters = StatsConfigParameters() configParameters.getAllParameters() machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags( configParameters.sourceMachinesTags) for machinePair in machinePairs: machinePair = str(machinePair).replace("[", "").replace( "]", "").replace(" ", "").replace("'", "").replace('"', '') #individual graphics commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) if generateTotalsGraphics == True: #print output commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def getTranslatorForModule( moduleAbsPath, language = None ): """ @summary : Returns a translator based the specified module and the language for which it is needed. @param moduleAbsPath: AbsolutePath name of the module for which we need the translation file. @param language: Language for whcih to find a proper translator. If none is specified, it will be set to the value found within the configuration file. @return: Return the translator to be used by _ . """ if language == None : configParameters = StatsConfigParameters() configParameters.getAllParameters() language = configParameters.mainApplicationLanguage fileName = LanguageTools.getTranslationFileName(language, moduleAbsPath) translator = LanguageTools.getTranslator(fileName) return translator
def main(): """ @summary : Small test case scenario allows for unit-like testing of the LanguageTools class. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() language = configParameters.mainApplicationLanguage paths = StatsPaths() paths.setBasicPaths() print "Language set in config file : %s" %language print "Test1 : (Should show that the proper translation file will be used) " fileName = LanguageTools.getTranslationFileName( language, paths.STATSLIB + 'StatsPlotter' ) print "Translation file to be used : %s " %( fileName ) print "Test2 : (Should translate the word into the specified language) " translator = LanguageTools.getTranslator( fileName ) print "Translation for bytecount : %s" %( translator("bytecount") ) print "Test3 : (Should be the same result as test 2) " translator = LanguageTools.getTranslatorForModule( paths.STATSLIB + 'StatsPlotter', language ) print "Translation for bytecount : %s" %( translator("bytecount") ) print "Test4 : Unless translation changes, this should print 'filecount' " print "Result : ", LanguageTools.translateTerm("nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py" )
def calculateTotalsForEachColumn( lines, includeGroups = False ): """ @summary : Goes through all the lines and makes the total of every field @param lines: Lines that we need to browse through. @return : list of totals for each fields. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups totals = [0.0 for i in range( len( lines[0].split(',' ) ) -1 ) ] for i in range( len( lines[1:] ) ): if lines[i+1].split(',')[0].split(' ')[0] not in knownGroups: #print "original line : " + lines[i+1] values = lines[i+1].split(',')[1:] #print "split up values : %s " %(values) for j in range(len(values)): totals[j] = totals[j] + float(values[j]) return totals
def transferLogFiles(): """ @summary : Log files will not be tansferred if local machine is not designed to be a pickling machine. If log files are to be transferred, they will be straight from the source." """ paths = StatsPaths() paths.setPaths() parameters = StatsConfigParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() parameters.getAllParameters() individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags ) individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines ) for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) : if picklingMachine == LOCAL_MACHINE :#pickling to be done here userName = machineParameters.getUserNameForMachine(sourceMachine) remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName ) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine ) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) ) print output
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) missingDays.append(self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays[1:]: self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay) self.__generateAllForDailyWebPage(False, generateTotalsGraphics) self.__generateAllGraphicsForGroups("daily") self.timeOfRequest = oldTimeOfRequest
def main(): """ @summary : Small test case scenario allows for unit-like testing of the LanguageTools class. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() language = configParameters.mainApplicationLanguage paths = StatsPaths() paths.setBasicPaths() print "Language set in config file : %s" % language print "Test1 : (Should show that the proper translation file will be used) " fileName = LanguageTools.getTranslationFileName( language, paths.STATSLIB + 'StatsPlotter') print "Translation file to be used : %s " % (fileName) print "Test2 : (Should translate the word into the specified language) " translator = LanguageTools.getTranslator(fileName) print "Translation for bytecount : %s" % (translator("bytecount")) print "Test3 : (Should be the same result as test 2) " translator = LanguageTools.getTranslatorForModule( paths.STATSLIB + 'StatsPlotter', language) print "Translation for bytecount : %s" % (translator("bytecount")) print "Test4 : Unless translation changes, this should print 'filecount' " print "Result : ", LanguageTools.translateTerm( "nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py")
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics): """ @summary : Generates the weekly graphics that were not generated between last update and timeOfRequest @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingWeek in missingWeeks: self.timeOfRequest = missingWeek self.__generateAllRRDGraphicsForWebPage("weekly", generateTotalsGraphics) self.__generateAllGraphicsForGroups("weekly") self.timeOfRequest = oldTimeOfRequest
def updateLogFiles(): """ @summary : Downloads the log files from the source machines into the local machine. """ os.system( "clear" ) showPresentation() print "" print "" print "Updating log files...This may take a while...." configParameters = StatsConfigParameters( ) configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() for tag in configParameters.sourceMachinesTags: sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for sourceMachine in sourceMachines: for i in range(3):#do 3 times in case of currently turning log files. status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) ) #print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) #print output time.sleep( 10 )
def updateLogFiles(): """ @summary : Downloads the log files from the source machines into the local machine. """ os.system("clear") showPresentation() print "" print "" print "Updating log files...This may take a while...." configParameters = StatsConfigParameters() configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() for tag in configParameters.sourceMachinesTags: sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for sourceMachine in sourceMachines: for i in range( 3): #do 3 times in case of currently turning log files. status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (machineParameters.getUserNameForMachine(sourceMachine), sourceMachine, StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine)) #print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) #print output time.sleep(10)
def addTotalsAndMeansToLines(lines): """ @summary : Calculates the total of every field found in the spreadsheet and add the line containing the totals to the lines received as a parameter. @param lines: List of lines contained in the spreadsheet we want to calculate the totals for. @return : Returns the lines with the totals line appended to the list of lines. """ global _ #totals section lineToAdd = 'Total ( without groups )' totals = calculateTotalsForEachColumn(lines) splitFirstLine = lines[0].split(',') for i in range(1, len(splitFirstLine)): if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str( splitFirstLine[i]).lower(): lineToAdd = lineToAdd + ',' + str(totals[i - 1]) else: lineToAdd = lineToAdd + ',' lines.append(lineToAdd) #means section configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups nbSourlients = getNbSourlients(lines, False) #print "nbsourlients :%s" %nbSourlients means = [0 for i in range(len(lines[0].split(',')) - 1)] for i in range(len(totals)): means[i] = float(totals[i]) / float(nbSourlients) lines.append( _('Means ( without groups ),') + str(means).replace('[', '').replace(']', '')) return lines
def __generateAllGraphicsForGroups(self, graphicType): """ @summary : Generated groups graphics based on the specified graphicType. @summary graphicType : "daily", "weekly", "monthly", "yearly" @raise Exception: When graphicType is unknown. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() supportedGraphicTypes = { "daily": "-d", "weekly": "-w", "monthly": "-m", "yearly": "-y" } if graphicType not in supportedGraphicTypes: raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" ) else: for group in configParameters.groupParameters.groups: groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat( group) groupMachines = str(groupMachines).replace("[", "").replace( "]", "").replace("'", "").replace('"', '') if graphicType == "daily": commands.getstatusoutput( '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s' % (self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage)) #print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage ) else: commands.getoutput( "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" % (self.paths.STATSBIN, supportedGraphicTypes[graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage)) print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" % ( self.paths.STATSBIN, supportedGraphicTypes[graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage)
def addTotalsAndMeansToLines( lines ): """ @summary : Calculates the total of every field found in the spreadsheet and add the line containing the totals to the lines received as a parameter. @param lines: List of lines contained in the spreadsheet we want to calculate the totals for. @return : Returns the lines with the totals line appended to the list of lines. """ global _ #totals section lineToAdd = 'Total ( without groups )' totals = calculateTotalsForEachColumn( lines ) splitFirstLine = lines[0].split(',') for i in range(1, len( splitFirstLine ) ): if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str(splitFirstLine[i]).lower(): lineToAdd = lineToAdd + ',' + str(totals[i-1]) else: lineToAdd = lineToAdd + ',' lines.append( lineToAdd ) #means section configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups nbSourlients = getNbSourlients( lines, False ) #print "nbsourlients :%s" %nbSourlients means = [0 for i in range( len( lines[0].split(',') ) -1 ) ] for i in range( len( totals ) ): means[i] = float(totals[i]) / float( nbSourlients ) lines.append( _('Means ( without groups ),') + str(means).replace( '[', '' ).replace( ']', '' ) ) return lines
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self): """ @summary : Generates all the csv files required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ #costs yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS monthlyCosts = yearlyCosts / 12.0 weeklyCosts = yearlyCosts / 52.0 #Get params from configuration files configParameters = StatsConfigParameters() configParameters.getAllParameters() clusters = str(configParameters.sourceMachinesTags).replace( '[', '').replace(']', '').replace(' ', '').replace('"', '').replace("'", "") updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForDailyWebPage(True, clusters, 0) if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts) if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts) if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllRRDGraphicsForWebPage( self, graphicType, generateTotalsGraphics = True ): """ @summary : This method generates new rrd graphics based on the specified graphics @param graphicType : daily weekly monthly or yearly @raise Exception : When graphicType is unknown. """ supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" } if graphicType not in supportedGraphicTypes: raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" ) configParameters = StatsConfigParameters( ) configParameters.getAllParameters() machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags(configParameters.sourceMachinesTags) for machinePair in machinePairs: machinePair = str(machinePair).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace( "'", "" ).replace( '"','' ) #individual graphics commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) if generateTotalsGraphics == True : #print output commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) ) # print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ # %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def generateTopWebPage(self): """ @summary : Generates the top web page based on the """ configParameters = StatsConfigParameters() configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() supportedLanguages = LanguageTools.getSupportedLanguages() self.__createTheWebPage( configParameters.sourceMachinesTags, supportedLanguages )
def getMainApplicationLanguage(): """ @summary : Reads and returns the main application language form the config file. @return : Le main application language. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() return configParameters.mainApplicationLanguage
def generateTopWebPage(self): """ @summary : Generates the top web page based on the """ configParameters = StatsConfigParameters() configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() supportedLanguages = LanguageTools.getSupportedLanguages() self.__createTheWebPage(configParameters.sourceMachinesTags, supportedLanguages)
def updatePickleFiles(infos): """ @summary : Updates pickles files from the specified start time to the specified end time. @param infos : @note : If update is not up to now, we presume that updating log files could cause us to loose precious log files. Therefore we update log files only if update is up to now, where we absolutely need recent log files. """ needToupdateLogFiles = askUserAboutUpdatingLogs(infos) if needToupdateLogFiles == True: updateLogFiles() configParameters = StatsConfigParameters() configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() os.system("clear") showPresentation() print "" print "" print "Updating pickles....This may take a while..." print "" for tag in configParameters.sourceMachinesTags: sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for sourceMachine in sourceMachines: status, output = commands.getstatusoutput( "python %spickleUpdater.py -f rx -m %s " % (StatsPaths.STATSBIN, sourceMachine)) #print output #print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine ) print "Updated rx pickles for : %s" % (sourceMachine) status, output = commands.getstatusoutput( "python %spickleUpdater.py -f tx -m %s " % (StatsPaths.STATSBIN, sourceMachine)) #print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine ) #print output print "Updated tx pickles for : %s" % (sourceMachine)
def updatePickleFiles( infos ): """ @summary : Updates pickles files from the specified start time to the specified end time. @param infos : @note : If update is not up to now, we presume that updating log files could cause us to loose precious log files. Therefore we update log files only if update is up to now, where we absolutely need recent log files. """ needToupdateLogFiles = askUserAboutUpdatingLogs( infos ) if needToupdateLogFiles == True : updateLogFiles() configParameters = StatsConfigParameters( ) configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() os.system( "clear" ) showPresentation() print "" print "" print "Updating pickles....This may take a while..." print "" for tag in configParameters.sourceMachinesTags: sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for sourceMachine in sourceMachines: status, output = commands.getstatusoutput( "python %spickleUpdater.py -f rx -m %s "%( StatsPaths.STATSBIN, sourceMachine ) ) #print output #print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine ) print "Updated rx pickles for : %s" %(sourceMachine) status, output = commands.getstatusoutput( "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN, sourceMachine) ) #print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine ) #print output print "Updated tx pickles for : %s" %(sourceMachine)
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self): """ @summary : Gets all the graphics required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ configParameters = StatsConfigParameters() configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup") requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } #-------------------- print "time of the request : ", self.timeOfRequest # print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForDailyWebPage(True, True, True) # print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency]( self.timeOfRequest) == True: #print "weeklies need to be updated." self.generateAllForWeeklyWebPage(True, True) # print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForMonthlyWebPage(True, True) # print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency]( self.timeOfRequest) == True: self.generateAllForYearlyWebPage(True, True)
def main(): """ @summary: Small test case to see if everything works fine """ statsConfig = StatsConfigParameters() statsConfig.getAllParameters() machineconfig = MachineConfigParameters() machineconfig.getParametersFromMachineConfigurationFile() currentTimeEpochFormat = time.time() - (120 * 60) endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat)) startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat - (StatsDateLib.DAY * 7))) print startTime, endTime groupName = statsConfig.groupParameters.groups[0] clients = statsConfig.groupParameters.groupsMembers[groupName] machines = statsConfig.groupParameters.groupsMachines[groupName] fileType = statsConfig.groupParameters.groupFileTypes[groupName] seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime=startTime, width=StatsDateLib.DAY * 7, interval=StatsDateLib.HOUR)[:-1]) listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators) listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles) currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555) if MemoryManagement.getTotalSizeListOfFiles( listOfFiles) > currentFreeMemory: seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes) print seperators else: print "We have %s bytes free and the pickles require %s bytes" % ( currentFreeMemory, getTotalSizeListOfFiles(listOfFiles)) print "we have enough memory to merge all these pickles."
def main(): """ @summary: Calls up the different methods required to set up the interface. """ if len(sys.argv) == 2: if sys.argv[1] == "-h" or sys.argv[1] == "--help": printHelp() else: path = sys.argv[1] currentlyUsedLanguages = [] configParameters = StatsConfigParameters() configParameters.getAllParameters() for languagePair in configParameters.webPagesLanguages: if languagePair[0] not in currentlyUsedLanguages: currentlyUsedLanguages.append(languagePair[0]) try: if not isValidRootInstallationPath(path): raise createRootFolderIfNecessary(path) copySourceFiles(currentlyUsedLanguages) createSubFolders(path, currentlyUsedLanguages) createSymbolicLinks(path, currentlyUsedLanguages) giveOutPermissionsToFolders(currentlyUsedLanguages) except: print _( "Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'." ) sys.exit() else: print _( "Error. Application must be called with one and only one parameter. Use -h|--help for further help." )
def getGroups( fileType, machine): """ @summary : Gathers group found in config file. @param fileType: Filetype for wich to search groups for. @param machine : Machien for wich to search groups for. @return: returns the list of groups matching the filetype and machine parameters """ configParameters = StatsConfigParameters() configParameters.getAllParameters() interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine( fileType, machine ) return interestingGroups
def runPickleTransfersToRRDDatabases( infos ): """ @summary : Runs the transfer from pickles to rrd databases from the start times found in the backup being used and until the specified end time. @param infos : """ os.system( "clear" ) showPresentation() print "" print "Updating databases...This may take a while..." print "" parameters = StatsConfigParameters( ) parameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() for tag in parameters.machinesToBackupInDb : machines = machineParameters.getMachinesAssociatedWith(tag) machines = str( machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) ) #print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) #print "output:%s" %output print "Databases were updated for the following cluster : %s" %( tag ) if parameters.groupParameters.groups != []: for group in parameters.groupParameters.groups : groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupMachines = str( parameters.groupParameters.groupsMachines[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupProducts = str( parameters.groupParameters.groupsProducts[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupFileTypes = str(parameters.groupParameters.groupFileTypes[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) ) #print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) #print output print "Databases were updated for the following group : %s " %( group )
def main(): """ @summary : This program is to be used to backup rrd databases and their corresponding time of update files. Backing up rrd databases at various point in time is a recommended paractice in case newly entered data is not valid. """ setGlobalLanguageParameters() timeToRestore = "2006-10-23 09:00:00" currentTime = time.time() currentTime = StatsDateLib.getIsoFromEpoch(currentTime) currentTime = StatsDateLib.getIsoWithRoundedSeconds(currentTime) currentTime = currentTime.replace(" ", "_") generalParameters = StatsConfigParameters() generalParameters.getAllParameters() if len(sys.argv) == 2: print sys.argv #try: timeToRestore = sys.argv[1] t = time.strptime( timeToRestore, '%Y-%m-%d %H:%M:%S') #will raise exception if format is wrong. split = timeToRestore.split() timeToRestore = "%s_%s" % (split[0], split[1]) # except: # print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"' # print "Program terminated." # sys.exit() restoreDatabaseUpdateTimes(timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep) restoreDatabases(timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep) else: print _("You must specify a date.") print _("Date must be of the folowing format YYYY-MM-DD HH:MM:SS") print _("Program terminated.")
def getGroups(fileType, machine): """ @summary : Gathers group found in config file. @param fileType: Filetype for wich to search groups for. @param machine : Machien for wich to search groups for. @return: returns the list of groups matching the filetype and machine parameters """ configParameters = StatsConfigParameters() configParameters.getAllParameters() interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine( fileType, machine) return interestingGroups
def main(): """ @summary: Calls up the different methods required to set up the interface. """ if len( sys.argv ) == 2: if sys.argv[1] == "-h" or sys.argv[1] == "--help": printHelp() else: path = sys.argv[1] currentlyUsedLanguages = [] configParameters = StatsConfigParameters() configParameters.getAllParameters() for languagePair in configParameters.webPagesLanguages: if languagePair[0] not in currentlyUsedLanguages: currentlyUsedLanguages.append( languagePair[0] ) try: if not isValidRootInstallationPath( path ) : raise createRootFolderIfNecessary( path ) copySourceFiles( currentlyUsedLanguages ) createSubFolders( path, currentlyUsedLanguages ) createSymbolicLinks( path, currentlyUsedLanguages ) giveOutPermissionsToFolders( currentlyUsedLanguages ) except : print _("Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'.") sys.exit() else: print _("Error. Application must be called with one and only one parameter. Use -h|--help for further help.")
def main(): """ @summary : This program is to be used to backup rrd databases and their corresponding time of update files. Backing up rrd databases at various point in time is a recommended paractice in case newly entered data is not valid. """ setGlobalLanguageParameters() timeToRestore = "2006-10-23 09:00:00" currentTime = time.time() currentTime = StatsDateLib.getIsoFromEpoch( currentTime ) currentTime = StatsDateLib.getIsoWithRoundedSeconds( currentTime ) currentTime = currentTime.replace(" ", "_") generalParameters = StatsConfigParameters() generalParameters.getAllParameters() if len( sys.argv ) == 2: print sys.argv #try: timeToRestore = sys.argv[1] t = time.strptime( timeToRestore, '%Y-%m-%d %H:%M:%S' )#will raise exception if format is wrong. split = timeToRestore.split() timeToRestore = "%s_%s" %( split[0], split[1] ) # except: # print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"' # print "Program terminated." # sys.exit() restoreDatabaseUpdateTimes( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep ) restoreDatabases( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep ) else: print _( "You must specify a date." ) print _( "Date must be of the folowing format YYYY-MM-DD HH:MM:SS" ) print _( "Program terminated." )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(self): """ @summary : Generates all the csv files required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ #costs yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS monthlyCosts = yearlyCosts / 12.0 weeklyCosts = yearlyCosts / 52.0 #Get params from configuration files configParameters = StatsConfigParameters( ) configParameters.getAllParameters() clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","") updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForDailyWebPage( True, clusters,0 ) if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts ) if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts ) if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def getCurrentlyActiveMachine(): """ @return: Returns the list of currently active source machines found within the config file. """ machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() configParameters = StatsConfigParameters() configParameters.getAllParameters() currentlyActiveMachines=[] for tag in configParameters.sourceMachinesTags: currentlyActiveMachines.extend( machineParameters.getMachinesAssociatedWith(tag) ) currentlyActiveMachines.extend( [concat ( machineParameters.getMachinesAssociatedWith(tag)) for tag in configParameters.sourceMachinesTags ] ) return currentlyActiveMachines
def getNbSourlients( lines, includeGroups = False ): """ @summary : Goes through the received lines and counts the number of sourlients found. @param lines :Lines to browse @param includeGroups : Whether to include groups or not. @return : Returns the number of sourlients found. """ global _ nbSourlients = 0 configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups for line in lines : entryIsValid = True fields = line.split( ',' ) if includeGroups == False: if str(fields[0].split( ' ' )[0]).replace(' ', '') in knownGroups : entryIsValid = False if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \ or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() : entryIsValid = False if entryIsValid == True: nbSourlients = nbSourlients + 1 return nbSourlients
def generateAllForEverySupportedWebPages(self): """ @summary : Generates all the csv files required by the web pages no matter what frequencies are found within the config file. """ yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS monthlyCosts = yearlyCosts / 12.0 weeklyCosts = yearlyCosts / 52.0 #Get params from configuration files configParameters = StatsConfigParameters( ) configParameters.getAllParameters() clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","") self.generateAllForDailyWebPage( True, clusters, 0 ) self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts ) self.generateAllForMonthlyWebPage( True, clusters, monthlyCosts ) self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self ): """ @summary : Gets all the graphics required by the web pages based on the update frequencies found within the config file. @note: Supposes that the web pages will require graphics from all the machines specified in the configuration file. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\ "monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear } #-------------------- print "time of the request : ", self.timeOfRequest # print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForDailyWebPage( True, True, True ) # print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True : #print "weeklies need to be updated." self.generateAllForWeeklyWebPage( True, True ) # print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForMonthlyWebPage( True, True ) # print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True : self.generateAllForYearlyWebPage( True, True )
def updateWebPages(generalParameters): """ @summary : Generates all the required web pages based on the language parameters found within the configuration files. """ paths = StatsPaths() paths.setPaths() generalParameters = StatsConfigParameters() generalParameters.getAllParameters() otherLanguages = [] generatorsTypes = [ DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator, MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator, TotalsGraphicsWebPageGenerator ] for languagePair in generalParameters.webPagesLanguages: for generatorsType in generatorsTypes: generator = generatorsType(languagePair[0], languagePair[1]) generator.generateWebPage() topWebPageGenerator = TopWebPageGenerator(languagePair[0]) topWebPageGenerator.generateTopWebPage() otherLanguages.append(languagePair[0]) try: while (1): otherLanguages.remove(generalParameters.mainApplicationLanguage) except: pass bottomWebPageGenerator = BottomWebPageGenerator( generalParameters.mainApplicationLanguage, otherLanguages) bottomWebPageGenerator.printWebPage()
def getNbSourlients(lines, includeGroups=False): """ @summary : Goes through the received lines and counts the number of sourlients found. @param lines :Lines to browse @param includeGroups : Whether to include groups or not. @return : Returns the number of sourlients found. """ global _ nbSourlients = 0 configParameters = StatsConfigParameters() configParameters.getAllParameters() knownGroups = configParameters.groupParameters.groups for line in lines: entryIsValid = True fields = line.split(',') if includeGroups == False: if str(fields[0].split(' ')[0]).replace(' ', '') in knownGroups: entryIsValid = False if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \ or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() : entryIsValid = False if entryIsValid == True: nbSourlients = nbSourlients + 1 return nbSourlients
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost): """ @summary : Generates the monthly graphics that were not generated between last update and timeOfRequest """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingYear in missingYears: self.timeOfRequest = missingYear self.__generateAllRRDGraphicsForWebPage( "yearly", True ) self.__generateAllGraphicsForGroups( "yearly" ) self.timeOfRequest = oldTimeOfRequest
def generateWebPage( self ): """ @summary : """ configParameters = StatsConfigParameters() configParameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() self.printWebPage( configParameters.sourceMachinesTags, machineParameters )
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) missingDays.append(self.timeOfRequest) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays[1:]: self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay ) self.__generateAllForDailyWebPage( False, generateTotalsGraphics ) self.__generateAllGraphicsForGroups( "daily" ) self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ): """ @summary : Generates the weekly graphics that were not generated between last update and timeOfRequest @param generateTotalsGraphics: Whether or not to generate the totals graphics. """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingWeek in missingWeeks: self.timeOfRequest = missingWeek self.__generateAllRRDGraphicsForWebPage( "weekly", generateTotalsGraphics ) self.__generateAllGraphicsForGroups( "weekly" ) self.timeOfRequest = oldTimeOfRequest
def transferLogFiles(): """ @summary : Log files will not be tansferred if local machine is not designed to be a pickling machine. If log files are to be transferred, they will be straight from the source." """ paths = StatsPaths() paths.setPaths() parameters = StatsConfigParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() parameters.getAllParameters() individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags) individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines) for sourceMachine, picklingMachine in map(None, individualSourceMachines, individualPicklingMachines): if picklingMachine == LOCAL_MACHINE: #pickling to be done here userName = machineParameters.getUserNameForMachine(sourceMachine) remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachine, userName) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % ( userName, sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (userName, sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine)) print output
def main(): """ @summary: Small test case to see if everything works fine """ statsConfig = StatsConfigParameters() statsConfig.getAllParameters() machineconfig = MachineConfigParameters() machineconfig.getParametersFromMachineConfigurationFile() currentTimeEpochFormat = time.time() -(120*60) endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) ) startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) ) print startTime, endTime groupName = statsConfig.groupParameters.groups[0] clients = statsConfig.groupParameters.groupsMembers[ groupName ] machines = statsConfig.groupParameters.groupsMachines[ groupName ] fileType = statsConfig.groupParameters.groupFileTypes[ groupName ] seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1]) listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators ) listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles) currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555) if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory: seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes ) print seperators else: print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) ) print "we have enough memory to merge all these pickles."
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile(): """ @summary : Reads the config file and returns all the currently running rx and tx names associated with any of the source machines found within the config file. @return: Returns the rxNames and the txNames found. """ rxNames = [] txNames = [] configParameters = StatsConfigParameters( ) configParameters.getAllParameters() for tag in configParameters.sourceMachinesTags: machine = configParameters.detailedParameters.sourceMachinesForTag[ tag ][0] newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine) rxNames.extend( filter( lambda x: x not in rxNames, newRxNames ) ) txNames.extend( filter( lambda x: x not in txNames, newTxNames ) ) return rxNames, txNames
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost ): """ @summary : generates the daily graphics that were not generated between last update and timeOfRequest. """ if clusters != [] and clusters != None: configParameters = StatsConfigParameters( ) configParameters.getAllParameters() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" ) missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest ) oldTimeOfRequest = self.timeOfRequest for missingDay in missingDays: self.timeOfRequest = missingDay self.__generateAllGraphicsForDailyWebPage( False, True ) self.timeOfRequest = oldTimeOfRequest
def updateWebPages( generalParameters ): """ @summary : Generates all the required web pages based on the language parameters found within the configuration files. """ paths = StatsPaths() paths.setPaths() generalParameters = StatsConfigParameters() generalParameters.getAllParameters() otherLanguages = [] generatorsTypes = [ DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator, MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator, TotalsGraphicsWebPageGenerator ] for languagePair in generalParameters.webPagesLanguages : for generatorsType in generatorsTypes : generator = generatorsType( languagePair[0], languagePair[1] ) generator.generateWebPage() topWebPageGenerator = TopWebPageGenerator(languagePair[0]) topWebPageGenerator.generateTopWebPage() otherLanguages.append( languagePair[0] ) try: while(1): otherLanguages.remove( generalParameters.mainApplicationLanguage ) except: pass bottomWebPageGenerator = BottomWebPageGenerator(generalParameters.mainApplicationLanguage, otherLanguages ) bottomWebPageGenerator.printWebPage()
def generateAllForEverySupportedWebPages(self): """ @summary : Generates all the csv files required by the web pages no matter what frequencies are found within the config file. """ yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS monthlyCosts = yearlyCosts / 12.0 weeklyCosts = yearlyCosts / 52.0 #Get params from configuration files configParameters = StatsConfigParameters() configParameters.getAllParameters() clusters = str(configParameters.sourceMachinesTags).replace( '[', '').replace(']', '').replace(' ', '').replace('"', '').replace("'", "") self.generateAllForDailyWebPage(True, clusters, 0) self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts) self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts) self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllForDailyWebPage( self, copyToColumbosFolder = True, generateTotalsGraphics = True ): """ @summary : Gets all the required daily graphs. @param getGraphicsMissingSinceLastUpdate : Whether or not to generate the daily graphics that did not get generated since the last update. @param generateTotalsGraphics : Whether or not to generate the graphics displaying the totals for each clusters. @todo : Add proper support for copyToColumbosFolder when generateAllGraphics finally support """ configParameters = StatsConfigParameters( ) configParameters.getAllParameters() machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(configParameters.sourceMachinesTags) for machineTag in configParameters.sourceMachinesTags: logins = [] machines = configParameters.detailedParameters.sourceMachinesForTag[machineTag] for machine in machines: logins.append( machineConfig.getUserNameForMachine(machine) ) logins = str(logins).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) machines = str(machines).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) if "," in machines : output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\ %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) ) #print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\ #%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) #print output else: output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) ) #print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) #print output if generateTotalsGraphics == True : for machinePair in machinePairs: #Generate all the daily total graphs. commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) ) #print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ # %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def __generateAllForDailyWebPage(self, copyToColumbosFolder=True, generateTotalsGraphics=True): """ @summary : Gets all the required daily graphs. @param getGraphicsMissingSinceLastUpdate : Whether or not to generate the daily graphics that did not get generated since the last update. @param generateTotalsGraphics : Whether or not to generate the graphics displaying the totals for each clusters. @todo : Add proper support for copyToColumbosFolder when generateAllGraphics finally support """ configParameters = StatsConfigParameters() configParameters.getAllParameters() machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags( configParameters.sourceMachinesTags) for machineTag in configParameters.sourceMachinesTags: logins = [] machines = configParameters.detailedParameters.sourceMachinesForTag[ machineTag] for machine in machines: logins.append(machineConfig.getUserNameForMachine(machine)) logins = str(logins).replace("[", "").replace("]", "").replace(" ", "") machines = str(machines).replace("[", "").replace("]", "").replace(" ", "") if "," in machines: output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\ %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) ) #print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\ #%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) #print output else: output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " % (self.paths.STATSBIN, machines.replace( "'", ""), logins.replace( "'", ""), self.timeOfRequest, self.outputLanguage)) #print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) #print output if generateTotalsGraphics == True: for machinePair in machinePairs: #Generate all the daily total graphs. commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) ) #print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ # %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\ %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def getParametersFromForm(self, form): """ @summary: Initialises the queryParameters based on the form received as parameter. @note : Absent parameters will be set to default values( [] or '' ) and will NOT raise exceptions. Use the searchForParameterErrors function to search for errors """ global _ #print form image = None #No image was produced yet #Every param is received in an array, use [0] to get first item, nothing for array. try: querier = form["querier"].replace("'", "").replace('"', '') except: querier = '' try: plotter = form["plotter"].replace("'", "").replace('"', '') except: plotter = '' try: fileTypes = form["fileType"].replace("'", "").replace('"', '') except: fileTypes = '' try: sourLients = form["sourLients"].split(',') except: sourLients = [] try: groupName = form["groupName"].replace("'", "").replace('"', '') except: groupName = '' try: machines = form["machines"].split(',') except: machines = [] if groupName != '' and (sourLients == [] or sourLients == ['']): configParameters = StatsConfigParameters() configParameters.getAllParameters() if groupName in configParameters.groupParameters.groups: if configParameters.groupParameters.groupFileTypes[ groupName] == fileTypes and configParameters.groupParameters.groupsMachines[ groupName] == machines: sourLients = configParameters.groupParameters.groupsMembers[ groupName] try: combine = form["combineSourlients"].replace(",", "").replace('"', '') if combine == 'false' or combine == 'False': combine = False elif combine == 'true' or combine == 'True': combine = True else: raise except: combine = False try: endTime = form["endTime"].replace("'", "").replace('"', '') hour = endTime.split(" ")[1] splitDate = endTime.split(" ")[0].split('-') endTime = "%s" % (splitDate[2] + '-' + splitDate[1] + '-' + splitDate[0] + " " + hour) if _("current") in str(form["fixedSpan"]).lower(): start, endTime = StatsDateLib.getStartEndFromCurrentDay( endTime) elif _("previous") in str(form["fixedSpan"]).lower(): start, endTime = StatsDateLib.getStartEndFromPreviousDay( endTime) except: endTime = '' try: products = form["products"].split(',') if products == [""]: raise except: products = ["*"] try: statsTypes = form["statsTypes"].split(',') except: statsTypes = [] #statsTypes = translateStatsTypes( statsTypes ) try: span = form["span"].replace("'", "").replace('"', '') if str(span).replace(' ', '') == '': raise span = int(span) except: span = 24 try: language = form["lang"].replace("'", "").replace('"', '') except: language = "" sourLients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters( endTime, span, sourLients, machines, [fileTypes]) self.queryParameters = GnuQueryBroker._QueryParameters( fileTypes, sourLients, groupName, machines, combine, endTime, products, statsTypes, span, language) self.replyParameters = GnuQueryBroker._ReplyParameters( querier, plotter, image, fileTypes, sourLients, groupName, machines, combine, endTime, products, statsTypes, span, '', language)
def runPickleTransfersToRRDDatabases(infos): """ @summary : Runs the transfer from pickles to rrd databases from the start times found in the backup being used and until the specified end time. @param infos : """ os.system("clear") showPresentation() print "" print "Updating databases...This may take a while..." print "" parameters = StatsConfigParameters() parameters.getAllParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() for tag in parameters.machinesToBackupInDb: machines = machineParameters.getMachinesAssociatedWith(tag) machines = str(machines).replace("[", "").replace("]", "").replace(" ", "") status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -m '%s' -e '%s'" % (StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime)) #print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) #print "output:%s" %output print "Databases were updated for the following cluster : %s" % (tag) if parameters.groupParameters.groups != []: for group in parameters.groupParameters.groups: groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "").replace("]", "").replace(" ", "") groupMachines = str( parameters.groupParameters.groupsMachines[group]).replace( "[", "").replace("]", "").replace(" ", "") groupProducts = str( parameters.groupParameters.groupsProducts[group]).replace( "[", "").replace("]", "").replace(" ", "") groupFileTypes = str( parameters.groupParameters.groupFileTypes[group]).replace( "[", "").replace("]", "").replace(" ", "") status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " % (StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts)) #print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) #print output print "Databases were updated for the following group : %s " % ( group)
def printWebPage(self, rxNames, txNames): """ @summary : Generates a web page based on all the rxnames and tx names that have run during the past x years. @param rxNames: List of sources for which to write links to their yearly graphics. @param txNames: List of clients for which to write links to their yearly graphics. @precondition: global _ translator must be set prior to calling this function. @notes : Only links to available graphics will be displayed. @return : None """ global _ rxNamesArray = rxNames.keys() txNamesArray = txNames.keys() rxNamesArray.sort() txNamesArray.sort() # Redirect output towards html page to generate. if not os.path.isdir(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML): os.makedirs(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML) fileHandle = open( "%syearlyGraphs_%s.html" % (self.pathsTowardsOutputFiles.STATSWEBPAGESHTML, self.displayedLanguage), "w" ) fileHandle.write( """ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> <link rel="stylesheet" href="../scripts/js_%s/windowfiles/dhtmlwindow.css" type="text/css" /> <script type="text/javascript" src="../scripts/js_%s/windowfiles/dhtmlwindow.js"> This is left here to give credit to the original creators of the dhtml script used for the group pop ups: /*********************************************** * DHTML Window Widget- Dynamic Drive (www.dynamicdrive.com) * This notice must stay intact for legal use. * Visit http://www.dynamicdrive.com/ for full source code ***********************************************/ </script> """ % (self.displayedLanguage, self.displayedLanguage) + """ <script type="text/javascript"> var descriptionWindow=dhtmlwindow.open("description", "inline", "description", "Description", "width=900px,height=120px,left=150px,top=10px,resize=1,scrolling=0", "recal") descriptionWindow.hide() </script> <head> <title> PX Graphics </title> <script> counter =0; function wopen(url, name, w, h){ // This function was taken on www.boutell.com w += 32; h += 96; counter +=1; var win = window.open(url, counter, 'width=' + w + ', height=' + h + ', ' + 'location=no, menubar=no, ' + 'status=no, toolbar=no, scrollbars=no, resizable=no'); win.resizeTo(w, h); win.focus(); } </script> <script> function showSourceHelpPage(){ var sourceHelpPage = dhtmlwindow.open("sourceHelpPage", "iframe", "helpPages/source_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'source'") + """", "width=875px,height=100px,resize=1,scrolling=1,center=1", "recal") sourceHelpPage.moveTo("middle", "middle"); } function showBytecountHelpPage(){ var byteCountHelpPage = dhtmlwindow.open("byteCount", "iframe", "helpPages/byteCount_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'byteCount'") + """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") byteCountHelpPage.moveTo("middle", "middle"); } function showClientHelpPage(){ var clientHelpPage = dhtmlwindow.open("client", "iframe", "helpPages/client_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'client'") + """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") .moveTo("middle", "middle"); } function showErrorsHelpPage(){ var errorsHelpPage = dhtmlwindow.open("errors", "iframe", "helpPages/errors_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'errors'") + """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") errorsHelpPage.moveTo("middle", "middle"); } function showFilecountHelpPage(){ var fileCountHelpPage = dhtmlwindow.open("fileCount", "iframe", "helpPages/fileCount_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'filecount'") + """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") fileCountHelpPage.moveTo("middle", "middle"); } function showFilesOverMaxLatencyHelpPage(){ var filesOverMaxLatencyHelpPage = dhtmlwindow.open("filesOverMaxLatency", "iframe", "helpPages/filesOverMaxLatency_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'filesOverMaxLatency'") + """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") filesOverMaxLatencyHelpPage.moveTo("middle", "middle"); } function showLatencyHelpPage(){ var latencyHelpPage = dhtmlwindow.open("latency", "iframe", "helpPages/latency_%s.html" """ % self.displayedLanguage + """, " """ + _("Definition of 'latency'") + """" , "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal") latencyHelpPage.moveTo("middle", "middle"); } </script> <STYLE> <!-- A{text-decoration:none} --> </STYLE> <style type="text/css"> div.left { float: left; } div.right {float: right; } </style> <style type="text/css"> a.blackLinks{ color: #000000; } div.tableContainer { width: 95%; /* table width will be 99% of this*/ height: 275px; /* must be greater than tbody*/ overflow: auto; margin: 0 auto; } table.cssTable { width: 99%; /*100% of container produces horiz. scroll in Mozilla*/ border: none; background-color: #f7f7f7; table-layout: fixed; } table.cssTable>tbody { /* child selector syntax which IE6 and older do not support*/ overflow: auto; height: 225px; overflow-x: hidden; } thead tr { position:relative; } thead td, thead th { text-align: center; font-size: 14px; background-color:"#006699"; color: steelblue; font-weight: bold; border-top: solid 1px #d8d8d8; } td.cssTable { color: #000; padding-right: 2px; font-size: 12px; text-align: left; border-bottom: solid 1px #d8d8d8; border-left: solid 1px #d8d8d8; } tfoot td { text-align: center; font-size: 11px; font-weight: bold; background-color: papayawhip; color: steelblue; border-top: solid 2px slategray; } td:last-child {padding-right: 20px;} /*prevent Mozilla scrollbar from hiding cell content*/ </style> </head> <body text="#000000" link="#FFFFFF" vlink="000000" bgcolor="#FFF4E5" > <br> <table width = "100%"> <tr width = "100%"> <div class="left"><b><font size="5">""" + _("Yearly graphics for RX sources from MetPx.") + """ </font><font size = "2">""" + _("*updated monthly") + """</font></b></div> """ ) oneFileFound = False for year in self.years: parameters = StatsConfigParameters() parameters.getAllParameters() machinesStr = ( str(parameters.sourceMachinesTags) .replace("[", "") .replace("]", "") .replace(",", "") .replace("'", "") .replace('"', "") .replace(" ", "") ) currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = self.pathsTowardsGraphics.STATSCSVFILES + _("yearly/rx/%s/%s.csv") % (machinesStr, currentYear) webLink = _("csvFiles/yearly/rx/%s/%s.csv") % (machinesStr, currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): if oneFileFound == False: fileHandle.write( "<div class='right'><font size='2' color='black'" + _("CSV files") + " : " ) oneFileFound = True fileHandle.write("""<a href="%s" class="blackLinks">%.4s.csv </a>""" % (webLink, currentYear)) if oneFileFound == True: fileHandle.write( """ </font> </div> """ ) fileHandle.write( """ </tr> </td> </table> <br> <br> <div class="tableContainer"> <table class="cssTable"> <thead> <tr> <td bgcolor="#006699" class="cssTable"> <font color = "white"> <center> """ + _("Sources") + """ <br> <a target ="popup" href="#" onClick="showSourceHelpPage(); return false;"> ? </a> <center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _("Display the total of bytes received every day of the week for each sources.") + """ "> <font color = "white"> <center> """ + _("Bytecount") + """ <br> <a target ="popup" href="#" onClick="showBytecountHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _("Display the total of files received every day of the week for each sources.") + """ "> <font color = "white"> <center> """ + _("Filecount") + """ <br> <a target ="popup" href="#" onClick="showFilecountHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title = " """ + _( "Display the total of errors that occured during the receptions for every day of the week for each sources." ) + """ "> <font color = "white"> <center> """ + _("Errors") + """ <br> <a target ="popup" href="#" onClick="showErrorsHelpPage(); return false;"> ? </a> </center> </font> </td> </tr> </thead> <tbody> """ ) for rxName in rxNamesArray: if rxNames[rxName] == "": fileHandle.write("""<tr> <td bgcolor="#99FF99" class="cssTable"> %s </td> """ % (rxName)) fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) else: machineName = self.getMachineNameFromDescription(rxNames[rxName]) fileHandle.write( """<tr> <td bgcolor="#99FF99" class="cssTable"><div class="left"> %s </div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """ % (rxName, rxNames[rxName].replace("'", "").replace('"', ""), machineName) ) fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/rx/%s/") % (rxName) + _("bytecount/%s.png") % str(currentYear) ) webLink = _("archives/yearly/rx/%s/") % (rxName) + _("bytecount/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (rxName, webLink, time.strftime("%Y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/rx/%s/") % (rxName) + _("filecount/%s.png") % str(currentYear) ) webLink = _("archives/yearly/rx/%s/") % (rxName) + _("filecount/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (rxName, webLink, time.strftime("%Y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/rx/%s/") % (rxName) + _("errors/%s.png") % str(currentYear) ) webLink = _("archives/yearly/rx/%s/") % (rxName) + _("errors/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (rxName, file, time.strftime("%Y", time.gmtime(year))) ) fileHandle.write("</td></tr>") fileHandle.write( """ </tbody> </table> </div> <br> <table width = "100%"> <tr width = "100%"> <div class="left"><b><font size="5">""" + _("Yearly graphics for TX clients from MetPx.") + """ </font><font size = "2">""" + _("*updated monthly") + """</font></b></div> """ ) oneFileFound = False for year in self.years: parameters = StatsConfigParameters() parameters.getAllParameters() machinesStr = ( str(parameters.sourceMachinesTags) .replace("[", "") .replace("]", "") .replace(",", "") .replace("'", "") .replace('"', "") .replace(" ", "") ) currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = self.pathsTowardsGraphics.STATSCSVFILES + _("yearly/tx/%s/%s.csv") % (machinesStr, currentYear) webLink = _("csvFiles/yearly/tx/%s/%s.csv") % (machinesStr, currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) print file if os.path.isfile(file): if oneFileFound == False: fileHandle.write( "<div class='right'><font size='2' color='black'>" + _("CSV files") + " : " ) oneFileFound = True fileHandle.write("""<a href="%s" class="blackLinks">%.4s.csv </a>""" % (webLink, currentYear)) if oneFileFound == True: fileHandle.write( """ </font> </div> """ ) fileHandle.write( """ </tr> </td> </table> <br> <br> <div class="tableContainer"> <table class="cssTable" > <thead> <tr> <td bgcolor="#006699" class="cssTable"> <font color = "white"> <center> """ + _("Clients") + """ <br> <a target ="popup" href="#" onClick="showClientHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _("Display the taverage latency of file transfers for every day of the week for each clients.") + """> <font color = "white"> <center> """ + _("Latency") + """ <br> <a target ="popup" href="#" onClick="showLatencyHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _( "Display the total number of files for wich the latency was over 15 seconds for every day of the week for each clients." ) + """> <font color = "white"> <center> """ + _("Files Over Max. Lat.") + """ <br> <a target ="popup" href="#" onClick="showFilesOverMaxLatencyHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _("Display the total of bytes transfered every day of the week for each clients.") + """> <font color = "white"> <center> """ + _("Bytecount") + """ <br> <a target ="popup" href="#" onClick="showBytecountHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _("Display the total of files transferred every day of the week for each clients.") + """> <font color = "white"> <center> """ + _("Filecount") + """ <br> <a target ="popup" href="#" onClick="showFilecountHelpPage(); return false;"> ? </a> </center> </font> </td> <td bgcolor="#006699" class="cssTable" title =" """ + _( "Display the total of errors that occured during file transfers every day of the week for each clients." ) + """> <font color = "white"> <center> """ + _("Errors") + """ <br> <a target ="popup" href="#" onClick="showErrorsHelpPage(); return false;"> ? </a> </center> </font> </td> </tr> </thead> <tbody> """ ) for txName in txNamesArray: if txNames[txName] == "": fileHandle.write(""" <tr> <td bgcolor="#99FF99" class="cssTable">%s</td> """ % (txName)) fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) else: machineName = self.getMachineNameFromDescription(txNames[txName]) fileHandle.write( """<tr> <td bgcolor="#99FF99" class="cssTable"><div class="left"> %s </div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """ % (txName, txNames[txName].replace("'", "").replace('"', ""), machineName) ) fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/tx/%s/") % (txName) + _("latency/%s.png") % str(currentYear) ) webLink = _("archives/yearly/tx/%s/") % (txName) + _("latency/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (txName, webLink, time.strftime("%y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/tx/%s/") % (txName) + _("filesOverMaxLatency/%s.png") % str(currentYear) ) webLink = _("archives/yearly/tx/%s/") % (txName) + _("filesOverMaxLatency/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (txName, webLink, time.strftime("%y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/tx/%s/") % (txName) + _("bytecount/%s.png") % str(currentYear) ) webLink = _("archives/yearly/tx/%s/") % (txName) + _("bytecount/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (txName, webLink, time.strftime("%y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF">""" + _("Years") + """ : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/tx/%s/") % (txName) + _("filecount/%s.png") % str(currentYear) ) webLink = _("archives/yearly/tx/%s/") % (txName) + _("filecount/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (txName, webLink, time.strftime("%y", time.gmtime(year))) ) fileHandle.write("</td>") fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >Years : """) for year in self.years: currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage) file = ( self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _("yearly/tx/%s/") % (txName) + _("errors/%s.png") % str(currentYear) ) webLink = _("archives/yearly/tx/%s/") % (txName) + _("errors/%s.png") % str(currentYear) _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage) if os.path.isfile(file): fileHandle.write( """<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>""" % (txName, webLink, time.strftime("%y", time.gmtime(year))) ) fileHandle.write("</td></tr>") fileHandle.write( """ </tbody> </table> </div> </body> </html> """ ) fileHandle.close()