def __generateAllMissingYearlyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingYears = updateManager.getMissingYearsBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage("yearly", True)
self.__generateAllGraphicsForGroups("yearly")
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage(False, True)
self.timeOfRequest = oldTimeOfRequest
def __generateAllGraphicsForGroups( self, graphicType ):
"""
@summary : Generated groups graphics based on the
specified graphicType.
@summary graphicType : "daily", "weekly", "monthly", "yearly"
@raise Exception: When graphicType is unknown.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" }
if graphicType not in supportedGraphicTypes:
raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" )
else:
for group in configParameters.groupParameters.groups:
groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat( group )
groupMachines = str(groupMachines).replace( "[", "" ).replace( "]", "" ).replace( "'", "" ).replace( '"','' )
if graphicType == "daily":
commands.getstatusoutput( '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage ) )
#print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage )
else:
commands.getoutput("%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage ) )
print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage )
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system("clear")
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(
3): #do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(machineParameters.getUserNameForMachine(sourceMachine),
sourceMachine, StatsPaths.PXLOG, StatsPaths.STATSLOGS,
sourceMachine))
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep(10)
def main():
"""
@summary : Small test case scenario allows
for unit-like testing of the LanguageTools
class.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
paths = StatsPaths()
paths.setBasicPaths()
print "Language set in config file : %s" % language
print "Test1 : (Should show that the proper translation file will be used) "
fileName = LanguageTools.getTranslationFileName(
language, paths.STATSLIB + 'StatsPlotter')
print "Translation file to be used : %s " % (fileName)
print "Test2 : (Should translate the word into the specified language) "
translator = LanguageTools.getTranslator(fileName)
print "Translation for bytecount : %s" % (translator("bytecount"))
print "Test3 : (Should be the same result as test 2) "
translator = LanguageTools.getTranslatorForModule(
paths.STATSLIB + 'StatsPlotter', language)
print "Translation for bytecount : %s" % (translator("bytecount"))
print "Test4 : Unless translation changes, this should print 'filecount' "
print "Result : ", LanguageTools.translateTerm(
"nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py")
def getTranslatorForModule( moduleAbsPath, language = None ):
"""
@summary : Returns a translator based the specified module
and the language for which it is needed.
@param moduleAbsPath: AbsolutePath name of the module for
which we need the translation file.
@param language: Language for whcih to find a proper translator.
If none is specified, it will be set to the value
found within the configuration file.
@return: Return the translator to be used by _ .
"""
if language == None :
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
fileName = LanguageTools.getTranslationFileName(language, moduleAbsPath)
translator = LanguageTools.getTranslator(fileName)
return translator
def main():
"""
@summary : Small test case scenario allows
for unit-like testing of the LanguageTools
class.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
paths = StatsPaths()
paths.setBasicPaths()
print "Language set in config file : %s" %language
print "Test1 : (Should show that the proper translation file will be used) "
fileName = LanguageTools.getTranslationFileName( language, paths.STATSLIB + 'StatsPlotter' )
print "Translation file to be used : %s " %( fileName )
print "Test2 : (Should translate the word into the specified language) "
translator = LanguageTools.getTranslator( fileName )
print "Translation for bytecount : %s" %( translator("bytecount") )
print "Test3 : (Should be the same result as test 2) "
translator = LanguageTools.getTranslatorForModule( paths.STATSLIB + 'StatsPlotter', language )
print "Translation for bytecount : %s" %( translator("bytecount") )
print "Test4 : Unless translation changes, this should print 'filecount' "
print "Result : ", LanguageTools.translateTerm("nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py" )
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile():
"""
@summary : Reads the config file and returns
all the currently running rx and tx names
associated with any of the source machines
found within the config file.
@return: Returns the rxNames and the txNames found.
"""
rxNames = []
txNames = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for tag in configParameters.sourceMachinesTags:
machine = configParameters.detailedParameters.sourceMachinesForTag[
tag][0]
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, machine)
rxNames.extend(filter(lambda x: x not in rxNames, newRxNames))
txNames.extend(filter(lambda x: x not in txNames, newTxNames))
return rxNames, txNames
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags )
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines )
for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) :
if picklingMachine == LOCAL_MACHINE :#pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName )
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine )
output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) )
print output
def getTranslatorForModule(moduleAbsPath, language=None):
"""
@summary : Returns a translator based the specified module
and the language for which it is needed.
@param moduleAbsPath: AbsolutePath name of the module for
which we need the translation file.
@param language: Language for whcih to find a proper translator.
If none is specified, it will be set to the value
found within the configuration file.
@return: Return the translator to be used by _ .
"""
if language == None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
fileName = LanguageTools.getTranslationFileName(
language, moduleAbsPath)
translator = LanguageTools.getTranslator(fileName)
return translator
def calculateTotalsForEachColumn( lines, includeGroups = False ):
"""
@summary : Goes through all the lines and
makes the total of every field
@param lines: Lines that we need to browse through.
@return : list of totals for each fields.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
totals = [0.0 for i in range( len( lines[0].split(',' ) ) -1 ) ]
for i in range( len( lines[1:] ) ):
if lines[i+1].split(',')[0].split(' ')[0] not in knownGroups:
#print "original line : " + lines[i+1]
values = lines[i+1].split(',')[1:]
#print "split up values : %s " %(values)
for j in range(len(values)):
totals[j] = totals[j] + float(values[j])
return totals
def calculateTotalsForEachColumn(lines, includeGroups=False):
"""
@summary : Goes through all the lines and
makes the total of every field
@param lines: Lines that we need to browse through.
@return : list of totals for each fields.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
totals = [0.0 for i in range(len(lines[0].split(',')) - 1)]
for i in range(len(lines[1:])):
if lines[i + 1].split(',')[0].split(' ')[0] not in knownGroups:
#print "original line : " + lines[i+1]
values = lines[i + 1].split(',')[1:]
#print "split up values : %s " %(values)
for j in range(len(values)):
totals[j] = totals[j] + float(values[j])
return totals
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(3):#do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) )
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep( 10 )
def __generateAllRRDGraphicsForWebPage(self,
graphicType,
generateTotalsGraphics=True):
"""
@summary : This method generates new rrd graphics
based on the specified graphics
@param graphicType : daily weekly monthly or yearly
@raise Exception : When graphicType is unknown.
"""
supportedGraphicTypes = {
"daily": "-d",
"weekly": "-w",
"monthly": "-m",
"yearly": "-y"
}
if graphicType not in supportedGraphicTypes:
raise Exception(
"Unsupported graphicType detected in __generateAllGraphicsForGroups"
)
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags(
configParameters.sourceMachinesTags)
for machinePair in machinePairs:
machinePair = str(machinePair).replace("[", "").replace(
"]", "").replace(" ", "").replace("'", "").replace('"', '')
#individual graphics
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
if generateTotalsGraphics == True:
#print output
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingWeeks = updateManager.getMissingWeeksBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage("weekly",
generateTotalsGraphics)
self.__generateAllGraphicsForGroups("weekly")
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingDailyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay)
self.__generateAllForDailyWebPage(False, generateTotalsGraphics)
self.__generateAllGraphicsForGroups("daily")
self.timeOfRequest = oldTimeOfRequest
def __init__( self, displayedLanguage = 'en', filesLanguage='en', days = None, \
weeks = None, months = None, years = None, \
pathsTowardsGraphics = None, pathsTowardsOutputFiles = None ):
"""
@summary : Constructor
@param displayedLanguage: Languages in which to display
the different captions found within
the generated web page.
@param fileLanguages: Language in which the files that
will be referenced within this page
have been generated.
@param days : List of days that the web page covers.
@note : Will set two global translators to be used throughout this module
_ which translates every caption that is to be printed.
_F which translates every filename that is to be linked.
"""
configParameters = StatsConfigParameters()
configParameters.getGeneralParametersFromStatsConfigurationFile()
global _
_ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, displayedLanguage )
if days == None:
self.setDays()
else:
self.days = days
if weeks == None:
self.setWeeks()
else:
self.weeks = weeks
if months == None:
self.setMonths()
else:
self.months = months
if years == None:
self.setYears()
else:
self.years = years
self.displayedLanguage = displayedLanguage
self.filesLanguage = filesLanguage
self.pathsTowardsGraphics = StatsPaths()
self.pathsTowardsGraphics.setPaths( filesLanguage )
self.pathsTowardsOutputFiles = StatsPaths()
self.pathsTowardsOutputFiles.setPaths( self.displayedLanguage )
StatsDateLib.setLanguage(filesLanguage)
def addTotalsAndMeansToLines(lines):
"""
@summary : Calculates the total of
every field found in the
spreadsheet and add the
line containing the totals
to the lines received as a
parameter.
@param lines: List of lines contained in
the spreadsheet we want to
calculate the totals for.
@return : Returns the lines with the
totals line appended to the
list of lines.
"""
global _
#totals section
lineToAdd = 'Total ( without groups )'
totals = calculateTotalsForEachColumn(lines)
splitFirstLine = lines[0].split(',')
for i in range(1, len(splitFirstLine)):
if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str(
splitFirstLine[i]).lower():
lineToAdd = lineToAdd + ',' + str(totals[i - 1])
else:
lineToAdd = lineToAdd + ','
lines.append(lineToAdd)
#means section
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
nbSourlients = getNbSourlients(lines, False)
#print "nbsourlients :%s" %nbSourlients
means = [0 for i in range(len(lines[0].split(',')) - 1)]
for i in range(len(totals)):
means[i] = float(totals[i]) / float(nbSourlients)
lines.append(
_('Means ( without groups ),') +
str(means).replace('[', '').replace(']', ''))
return lines
def __generateAllGraphicsForGroups(self, graphicType):
"""
@summary : Generated groups graphics based on the
specified graphicType.
@summary graphicType : "daily", "weekly", "monthly", "yearly"
@raise Exception: When graphicType is unknown.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
supportedGraphicTypes = {
"daily": "-d",
"weekly": "-w",
"monthly": "-m",
"yearly": "-y"
}
if graphicType not in supportedGraphicTypes:
raise Exception(
"Unsupported graphicType detected in __generateAllGraphicsForGroups"
)
else:
for group in configParameters.groupParameters.groups:
groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat(
group)
groupMachines = str(groupMachines).replace("[", "").replace(
"]", "").replace("'", "").replace('"', '')
if graphicType == "daily":
commands.getstatusoutput(
'%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s'
% (self.paths.STATSBIN, group, groupMembers,
self.timeOfRequest, groupMachines, groupFileTypes,
groupProducts, self.outputLanguage))
#print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage )
else:
commands.getoutput(
"%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s"
% (self.paths.STATSBIN,
supportedGraphicTypes[graphicType], groupFileTypes,
groupMachines, group, self.timeOfRequest,
self.outputLanguage))
print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" % (
self.paths.STATSBIN,
supportedGraphicTypes[graphicType], groupFileTypes,
groupMachines, group, self.timeOfRequest,
self.outputLanguage)
def addTotalsAndMeansToLines( lines ):
"""
@summary : Calculates the total of
every field found in the
spreadsheet and add the
line containing the totals
to the lines received as a
parameter.
@param lines: List of lines contained in
the spreadsheet we want to
calculate the totals for.
@return : Returns the lines with the
totals line appended to the
list of lines.
"""
global _
#totals section
lineToAdd = 'Total ( without groups )'
totals = calculateTotalsForEachColumn( lines )
splitFirstLine = lines[0].split(',')
for i in range(1, len( splitFirstLine ) ):
if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str(splitFirstLine[i]).lower():
lineToAdd = lineToAdd + ',' + str(totals[i-1])
else:
lineToAdd = lineToAdd + ','
lines.append( lineToAdd )
#means section
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
nbSourlients = getNbSourlients( lines, False )
#print "nbsourlients :%s" %nbSourlients
means = [0 for i in range( len( lines[0].split(',') ) -1 ) ]
for i in range( len( totals ) ):
means[i] = float(totals[i]) / float( nbSourlients )
lines.append( _('Means ( without groups ),') + str(means).replace( '[', '' ).replace( ']', '' ) )
return lines
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Generates all the csv files required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
#costs
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
clusters = str(configParameters.sourceMachinesTags).replace(
'[', '').replace(']', '').replace(' ',
'').replace('"',
'').replace("'", "")
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, clusters, 0)
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllRRDGraphicsForWebPage( self, graphicType, generateTotalsGraphics = True ):
"""
@summary : This method generates new rrd graphics
based on the specified graphics
@param graphicType : daily weekly monthly or yearly
@raise Exception : When graphicType is unknown.
"""
supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" }
if graphicType not in supportedGraphicTypes:
raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" )
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags(configParameters.sourceMachinesTags)
for machinePair in machinePairs:
machinePair = str(machinePair).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace( "'", "" ).replace( '"','' )
#individual graphics
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
if generateTotalsGraphics == True :
#print output
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def generateTopWebPage(self):
"""
@summary : Generates the top web page based on the
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
supportedLanguages = LanguageTools.getSupportedLanguages()
self.__createTheWebPage( configParameters.sourceMachinesTags, supportedLanguages )
def generateTopWebPage(self):
"""
@summary : Generates the top web page based on the
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
supportedLanguages = LanguageTools.getSupportedLanguages()
self.__createTheWebPage(configParameters.sourceMachinesTags,
supportedLanguages)
def updatePickleFiles(infos):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs(infos)
if needToupdateLogFiles == True:
updateLogFiles()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system("clear")
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f rx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" % (sourceMachine)
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f tx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" % (sourceMachine)
def getMainApplicationLanguage():
"""
@summary : Reads and returns the main application
language form the config file.
@return : Le main application language.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
return configParameters.mainApplicationLanguage
def getMainApplicationLanguage():
"""
@summary : Reads and returns the main application
language form the config file.
@return : Le main application language.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
return configParameters.mainApplicationLanguage
def updatePickleFiles( infos ):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs( infos )
if needToupdateLogFiles == True :
updateLogFiles()
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f rx -m %s "%( StatsPaths.STATSBIN, sourceMachine ) )
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" %(sourceMachine)
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN, sourceMachine) )
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" %(sourceMachine)
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Gets all the graphics required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
#-------------------- print "time of the request : ", self.timeOfRequest
# print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, True, True)
# print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
#print "weeklies need to be updated."
self.generateAllForWeeklyWebPage(True, True)
# print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, True)
# print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, True)
def filterGroupNames( fileName ):
"""
When called within pythons builtin
filter method will remove all entries
starting with a dot.
"""
statsConfigParameters = StatsConfigParameters()
statsConfigParameters.getGroupSettingsFromConfigurationFile()
groupNames = statsConfigParameters.groupParameters.groups
result = filter(lambda groupName: groupName in fileName, groupNames)
return result == []
def filterGroupNames(fileName):
"""
When called within pythons builtin
filter method will remove all entries
starting with a dot.
"""
statsConfigParameters = StatsConfigParameters()
statsConfigParameters.getGroupSettingsFromConfigurationFile()
groupNames = statsConfigParameters.groupParameters.groups
result = filter(lambda groupName: groupName in fileName, groupNames)
return result == []
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() - (120 * 60)
endTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat))
startTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat -
(StatsDateLib.DAY * 7)))
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[groupName]
machines = statsConfig.groupParameters.groupsMachines[groupName]
fileType = statsConfig.groupParameters.groupFileTypes[groupName]
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=StatsDateLib.DAY * 7,
interval=StatsDateLib.HOUR)[:-1])
listOfFiles = PickleMerging.createMergedPicklesList(
startTime, endTime, clients, groupName, fileType, machines, seperators)
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles(
listOfFiles) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentFreeMemory, listOfFileSizes)
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" % (
currentFreeMemory, getTotalSizeListOfFiles(listOfFiles))
print "we have enough memory to merge all these pickles."
def main():
"""
@summary: Calls up the different methods required to set up
the interface.
"""
if len(sys.argv) == 2:
if sys.argv[1] == "-h" or sys.argv[1] == "--help":
printHelp()
else:
path = sys.argv[1]
currentlyUsedLanguages = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for languagePair in configParameters.webPagesLanguages:
if languagePair[0] not in currentlyUsedLanguages:
currentlyUsedLanguages.append(languagePair[0])
try:
if not isValidRootInstallationPath(path):
raise
createRootFolderIfNecessary(path)
copySourceFiles(currentlyUsedLanguages)
createSubFolders(path, currentlyUsedLanguages)
createSymbolicLinks(path, currentlyUsedLanguages)
giveOutPermissionsToFolders(currentlyUsedLanguages)
except:
print _(
"Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'."
)
sys.exit()
else:
print _(
"Error. Application must be called with one and only one parameter. Use -h|--help for further help."
)
def getGroups( fileType, machine):
"""
@summary : Gathers group found in config file.
@param fileType: Filetype for wich to search groups for.
@param machine : Machien for wich to search groups for.
@return: returns the list of groups matching the filetype and machine parameters
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine( fileType, machine )
return interestingGroups
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch(currentTime)
currentTime = StatsDateLib.getIsoWithRoundedSeconds(currentTime)
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len(sys.argv) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime(
timeToRestore,
'%Y-%m-%d %H:%M:%S') #will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" % (split[0], split[1])
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
restoreDatabases(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
else:
print _("You must specify a date.")
print _("Date must be of the folowing format YYYY-MM-DD HH:MM:SS")
print _("Program terminated.")
def runPickleTransfersToRRDDatabases( infos ):
"""
@summary : Runs the transfer from pickles to rrd databases
from the start times found in the backup being used
and until the specified end time.
@param infos :
"""
os.system( "clear" )
showPresentation()
print ""
print "Updating databases...This may take a while..."
print ""
parameters = StatsConfigParameters( )
parameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in parameters.machinesToBackupInDb :
machines = machineParameters.getMachinesAssociatedWith(tag)
machines = str( machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) )
#print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime )
#print "output:%s" %output
print "Databases were updated for the following cluster : %s" %( tag )
if parameters.groupParameters.groups != []:
for group in parameters.groupParameters.groups :
groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupMachines = str( parameters.groupParameters.groupsMachines[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupProducts = str( parameters.groupParameters.groupsProducts[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupFileTypes = str(parameters.groupParameters.groupFileTypes[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) )
#print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts )
#print output
print "Databases were updated for the following group : %s " %( group )
def getGroups(fileType, machine):
"""
@summary : Gathers group found in config file.
@param fileType: Filetype for wich to search groups for.
@param machine : Machien for wich to search groups for.
@return: returns the list of groups matching the filetype and machine parameters
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine(
fileType, machine)
return interestingGroups
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentTime = StatsDateLib.getIsoWithRoundedSeconds( currentTime )
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len( sys.argv ) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime( timeToRestore, '%Y-%m-%d %H:%M:%S' )#will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" %( split[0], split[1] )
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
restoreDatabases( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
else:
print _( "You must specify a date." )
print _( "Date must be of the folowing format YYYY-MM-DD HH:MM:SS" )
print _( "Program terminated." )
def main():
"""
@summary: Calls up the different methods required to set up
the interface.
"""
if len( sys.argv ) == 2:
if sys.argv[1] == "-h" or sys.argv[1] == "--help":
printHelp()
else:
path = sys.argv[1]
currentlyUsedLanguages = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for languagePair in configParameters.webPagesLanguages:
if languagePair[0] not in currentlyUsedLanguages:
currentlyUsedLanguages.append( languagePair[0] )
try:
if not isValidRootInstallationPath( path ) :
raise
createRootFolderIfNecessary( path )
copySourceFiles( currentlyUsedLanguages )
createSubFolders( path, currentlyUsedLanguages )
createSymbolicLinks( path, currentlyUsedLanguages )
giveOutPermissionsToFolders( currentlyUsedLanguages )
except :
print _("Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'.")
sys.exit()
else:
print _("Error. Application must be called with one and only one parameter. Use -h|--help for further help.")
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(self):
"""
@summary : Generates all the csv files required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
#costs
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","")
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForDailyWebPage( True, clusters,0 )
if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts )
if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts )
if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def getCurrentlyActiveMachine():
"""
@return: Returns the list of currently active
source machines found within the config
file.
"""
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
currentlyActiveMachines=[]
for tag in configParameters.sourceMachinesTags:
currentlyActiveMachines.extend( machineParameters.getMachinesAssociatedWith(tag) )
currentlyActiveMachines.extend( [concat ( machineParameters.getMachinesAssociatedWith(tag)) for tag in configParameters.sourceMachinesTags ] )
return currentlyActiveMachines
def getNbSourlients( lines, includeGroups = False ):
"""
@summary : Goes through the received lines
and counts the number of sourlients
found.
@param lines :Lines to browse
@param includeGroups : Whether to include groups or not.
@return : Returns the number of sourlients found.
"""
global _
nbSourlients = 0
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
for line in lines :
entryIsValid = True
fields = line.split( ',' )
if includeGroups == False:
if str(fields[0].split( ' ' )[0]).replace(' ', '') in knownGroups :
entryIsValid = False
if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \
or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() :
entryIsValid = False
if entryIsValid == True:
nbSourlients = nbSourlients + 1
return nbSourlients
def generateAllForEverySupportedWebPages(self):
"""
@summary : Generates all the csv files required by
the web pages no matter what frequencies
are found within the config file.
"""
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","")
self.generateAllForDailyWebPage( True, clusters, 0 )
self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts )
self.generateAllForMonthlyWebPage( True, clusters, monthlyCosts )
self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self ):
"""
@summary : Gets all the graphics required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
#-------------------- print "time of the request : ", self.timeOfRequest
# print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForDailyWebPage( True, True, True )
# print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True :
#print "weeklies need to be updated."
self.generateAllForWeeklyWebPage( True, True )
# print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForMonthlyWebPage( True, True )
# print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForYearlyWebPage( True, True )
def updateWebPages(generalParameters):
"""
@summary : Generates all the required web pages
based on the language parameters found within
the configuration files.
"""
paths = StatsPaths()
paths.setPaths()
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
otherLanguages = []
generatorsTypes = [
DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator,
MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator,
TotalsGraphicsWebPageGenerator
]
for languagePair in generalParameters.webPagesLanguages:
for generatorsType in generatorsTypes:
generator = generatorsType(languagePair[0], languagePair[1])
generator.generateWebPage()
topWebPageGenerator = TopWebPageGenerator(languagePair[0])
topWebPageGenerator.generateTopWebPage()
otherLanguages.append(languagePair[0])
try:
while (1):
otherLanguages.remove(generalParameters.mainApplicationLanguage)
except:
pass
bottomWebPageGenerator = BottomWebPageGenerator(
generalParameters.mainApplicationLanguage, otherLanguages)
bottomWebPageGenerator.printWebPage()
def getNbSourlients(lines, includeGroups=False):
"""
@summary : Goes through the received lines
and counts the number of sourlients
found.
@param lines :Lines to browse
@param includeGroups : Whether to include groups or not.
@return : Returns the number of sourlients found.
"""
global _
nbSourlients = 0
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
for line in lines:
entryIsValid = True
fields = line.split(',')
if includeGroups == False:
if str(fields[0].split(' ')[0]).replace(' ', '') in knownGroups:
entryIsValid = False
if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \
or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() :
entryIsValid = False
if entryIsValid == True:
nbSourlients = nbSourlients + 1
return nbSourlients
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage( "yearly", True )
self.__generateAllGraphicsForGroups( "yearly" )
self.timeOfRequest = oldTimeOfRequest
def generateWebPage( self ):
"""
@summary :
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.printWebPage( configParameters.sourceMachinesTags, machineParameters )
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay )
self.__generateAllForDailyWebPage( False, generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "daily" )
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage( "weekly", generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "weekly" )
self.timeOfRequest = oldTimeOfRequest
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() -(120*60)
endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) )
startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) )
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[ groupName ]
machines = statsConfig.groupParameters.groupsMachines[ groupName ]
fileType = statsConfig.groupParameters.groupFileTypes[ groupName ]
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1])
listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators )
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes )
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) )
print "we have enough memory to merge all these pickles."
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.sourceMachinesTags)
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.picklingMachines)
for sourceMachine, picklingMachine in map(None, individualSourceMachines,
individualPicklingMachines):
if picklingMachine == LOCAL_MACHINE: #pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG,
sourceMachine, userName)
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (
userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine)
output = commands.getoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine))
print output
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage( False, True )
self.timeOfRequest = oldTimeOfRequest
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile():
"""
@summary : Reads the config file and returns
all the currently running rx and tx names
associated with any of the source machines
found within the config file.
@return: Returns the rxNames and the txNames found.
"""
rxNames = []
txNames = []
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
for tag in configParameters.sourceMachinesTags:
machine = configParameters.detailedParameters.sourceMachinesForTag[ tag ][0]
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine)
rxNames.extend( filter( lambda x: x not in rxNames, newRxNames ) )
txNames.extend( filter( lambda x: x not in txNames, newTxNames ) )
return rxNames, txNames
def updateWebPages( generalParameters ):
"""
@summary : Generates all the required web pages
based on the language parameters found within
the configuration files.
"""
paths = StatsPaths()
paths.setPaths()
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
otherLanguages = []
generatorsTypes = [ DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator, MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator, TotalsGraphicsWebPageGenerator ]
for languagePair in generalParameters.webPagesLanguages :
for generatorsType in generatorsTypes :
generator = generatorsType( languagePair[0], languagePair[1] )
generator.generateWebPage()
topWebPageGenerator = TopWebPageGenerator(languagePair[0])
topWebPageGenerator.generateTopWebPage()
otherLanguages.append( languagePair[0] )
try:
while(1):
otherLanguages.remove( generalParameters.mainApplicationLanguage )
except:
pass
bottomWebPageGenerator = BottomWebPageGenerator(generalParameters.mainApplicationLanguage, otherLanguages )
bottomWebPageGenerator.printWebPage()
def generateAllForEverySupportedWebPages(self):
"""
@summary : Generates all the csv files required by
the web pages no matter what frequencies
are found within the config file.
"""
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
clusters = str(configParameters.sourceMachinesTags).replace(
'[', '').replace(']', '').replace(' ',
'').replace('"',
'').replace("'", "")
self.generateAllForDailyWebPage(True, clusters, 0)
self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts)
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts)
self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllForDailyWebPage(self,
copyToColumbosFolder=True,
generateTotalsGraphics=True):
"""
@summary : Gets all the required daily graphs.
@param getGraphicsMissingSinceLastUpdate : Whether or not to generate
the daily graphics that did
not get generated since the
last update.
@param generateTotalsGraphics : Whether or not to generate the graphics
displaying the totals for each clusters.
@todo : Add proper support for copyToColumbosFolder
when generateAllGraphics finally support
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(
configParameters.sourceMachinesTags)
for machineTag in configParameters.sourceMachinesTags:
logins = []
machines = configParameters.detailedParameters.sourceMachinesForTag[
machineTag]
for machine in machines:
logins.append(machineConfig.getUserNameForMachine(machine))
logins = str(logins).replace("[", "").replace("]",
"").replace(" ", "")
machines = str(machines).replace("[",
"").replace("]",
"").replace(" ", "")
if "," in machines:
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) )
#print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
#%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
else:
output = commands.getoutput(
"%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s "
% (self.paths.STATSBIN, machines.replace(
"'", ""), logins.replace(
"'", ""), self.timeOfRequest, self.outputLanguage))
#print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
if generateTotalsGraphics == True:
for machinePair in machinePairs:
#Generate all the daily total graphs.
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) )
#print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage)
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
