def __generateAllMissingDailyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage(False, True)
self.timeOfRequest = oldTimeOfRequest
def __generateAllGraphicsForGroups( self, graphicType ):
"""
@summary : Generated groups graphics based on the
specified graphicType.
@summary graphicType : "daily", "weekly", "monthly", "yearly"
@raise Exception: When graphicType is unknown.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" }
if graphicType not in supportedGraphicTypes:
raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" )
else:
for group in configParameters.groupParameters.groups:
groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat( group )
groupMachines = str(groupMachines).replace( "[", "" ).replace( "]", "" ).replace( "'", "" ).replace( '"','' )
if graphicType == "daily":
commands.getstatusoutput( '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage ) )
#print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage )
else:
commands.getoutput("%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage ) )
print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" %( self.paths.STATSBIN, supportedGraphicTypes[ graphicType], groupFileTypes, groupMachines, group, self.timeOfRequest, self.outputLanguage )
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate(
self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingYears = updateManager.getMissingYearsBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage("yearly", True)
self.__generateAllGraphicsForGroups("yearly")
self.timeOfRequest = oldTimeOfRequest
def calculateTotalsForEachColumn(lines, includeGroups=False):
"""
@summary : Goes through all the lines and
makes the total of every field
@param lines: Lines that we need to browse through.
@return : list of totals for each fields.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
totals = [0.0 for i in range(len(lines[0].split(',')) - 1)]
for i in range(len(lines[1:])):
if lines[i + 1].split(',')[0].split(' ')[0] not in knownGroups:
#print "original line : " + lines[i+1]
values = lines[i + 1].split(',')[1:]
#print "split up values : %s " %(values)
for j in range(len(values)):
totals[j] = totals[j] + float(values[j])
return totals
def getTranslatorForModule(moduleAbsPath, language=None):
"""
@summary : Returns a translator based the specified module
and the language for which it is needed.
@param moduleAbsPath: AbsolutePath name of the module for
which we need the translation file.
@param language: Language for whcih to find a proper translator.
If none is specified, it will be set to the value
found within the configuration file.
@return: Return the translator to be used by _ .
"""
if language == None:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
fileName = LanguageTools.getTranslationFileName(
language, moduleAbsPath)
translator = LanguageTools.getTranslator(fileName)
return translator
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile():
"""
@summary : Reads the config file and returns
all the currently running rx and tx names
associated with any of the source machines
found within the config file.
@return: Returns the rxNames and the txNames found.
"""
rxNames = []
txNames = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for tag in configParameters.sourceMachinesTags:
machine = configParameters.detailedParameters.sourceMachinesForTag[
tag][0]
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, machine)
rxNames.extend(filter(lambda x: x not in rxNames, newRxNames))
txNames.extend(filter(lambda x: x not in txNames, newTxNames))
return rxNames, txNames
def __generateAllRRDGraphicsForWebPage(self,
graphicType,
generateTotalsGraphics=True):
"""
@summary : This method generates new rrd graphics
based on the specified graphics
@param graphicType : daily weekly monthly or yearly
@raise Exception : When graphicType is unknown.
"""
supportedGraphicTypes = {
"daily": "-d",
"weekly": "-w",
"monthly": "-m",
"yearly": "-y"
}
if graphicType not in supportedGraphicTypes:
raise Exception(
"Unsupported graphicType detected in __generateAllGraphicsForGroups"
)
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags(
configParameters.sourceMachinesTags)
for machinePair in machinePairs:
machinePair = str(machinePair).replace("[", "").replace(
"]", "").replace(" ", "").replace("'", "").replace('"', '')
#individual graphics
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
if generateTotalsGraphics == True:
#print output
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def getTranslatorForModule( moduleAbsPath, language = None ):
"""
@summary : Returns a translator based the specified module
and the language for which it is needed.
@param moduleAbsPath: AbsolutePath name of the module for
which we need the translation file.
@param language: Language for whcih to find a proper translator.
If none is specified, it will be set to the value
found within the configuration file.
@return: Return the translator to be used by _ .
"""
if language == None :
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
fileName = LanguageTools.getTranslationFileName(language, moduleAbsPath)
translator = LanguageTools.getTranslator(fileName)
return translator
def main():
"""
@summary : Small test case scenario allows
for unit-like testing of the LanguageTools
class.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
paths = StatsPaths()
paths.setBasicPaths()
print "Language set in config file : %s" %language
print "Test1 : (Should show that the proper translation file will be used) "
fileName = LanguageTools.getTranslationFileName( language, paths.STATSLIB + 'StatsPlotter' )
print "Translation file to be used : %s " %( fileName )
print "Test2 : (Should translate the word into the specified language) "
translator = LanguageTools.getTranslator( fileName )
print "Translation for bytecount : %s" %( translator("bytecount") )
print "Test3 : (Should be the same result as test 2) "
translator = LanguageTools.getTranslatorForModule( paths.STATSLIB + 'StatsPlotter', language )
print "Translation for bytecount : %s" %( translator("bytecount") )
print "Test4 : Unless translation changes, this should print 'filecount' "
print "Result : ", LanguageTools.translateTerm("nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py" )
def calculateTotalsForEachColumn( lines, includeGroups = False ):
"""
@summary : Goes through all the lines and
makes the total of every field
@param lines: Lines that we need to browse through.
@return : list of totals for each fields.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
totals = [0.0 for i in range( len( lines[0].split(',' ) ) -1 ) ]
for i in range( len( lines[1:] ) ):
if lines[i+1].split(',')[0].split(' ')[0] not in knownGroups:
#print "original line : " + lines[i+1]
values = lines[i+1].split(',')[1:]
#print "split up values : %s " %(values)
for j in range(len(values)):
totals[j] = totals[j] + float(values[j])
return totals
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags )
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines )
for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) :
if picklingMachine == LOCAL_MACHINE :#pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName )
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine )
output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) )
print output
def __generateAllMissingDailyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay)
self.__generateAllForDailyWebPage(False, generateTotalsGraphics)
self.__generateAllGraphicsForGroups("daily")
self.timeOfRequest = oldTimeOfRequest
def main():
"""
@summary : Small test case scenario allows
for unit-like testing of the LanguageTools
class.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
language = configParameters.mainApplicationLanguage
paths = StatsPaths()
paths.setBasicPaths()
print "Language set in config file : %s" % language
print "Test1 : (Should show that the proper translation file will be used) "
fileName = LanguageTools.getTranslationFileName(
language, paths.STATSLIB + 'StatsPlotter')
print "Translation file to be used : %s " % (fileName)
print "Test2 : (Should translate the word into the specified language) "
translator = LanguageTools.getTranslator(fileName)
print "Translation for bytecount : %s" % (translator("bytecount"))
print "Test3 : (Should be the same result as test 2) "
translator = LanguageTools.getTranslatorForModule(
paths.STATSLIB + 'StatsPlotter', language)
print "Translation for bytecount : %s" % (translator("bytecount"))
print "Test4 : Unless translation changes, this should print 'filecount' "
print "Result : ", LanguageTools.translateTerm(
"nbreDeFichiers", "fr", "en", paths.STATSLIB + "StatsPlotter.py")
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingWeeks = updateManager.getMissingWeeksBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage("weekly",
generateTotalsGraphics)
self.__generateAllGraphicsForGroups("weekly")
self.timeOfRequest = oldTimeOfRequest
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(3):#do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) )
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep( 10 )
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system("clear")
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(
3): #do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(machineParameters.getUserNameForMachine(sourceMachine),
sourceMachine, StatsPaths.PXLOG, StatsPaths.STATSLOGS,
sourceMachine))
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep(10)
def addTotalsAndMeansToLines(lines):
"""
@summary : Calculates the total of
every field found in the
spreadsheet and add the
line containing the totals
to the lines received as a
parameter.
@param lines: List of lines contained in
the spreadsheet we want to
calculate the totals for.
@return : Returns the lines with the
totals line appended to the
list of lines.
"""
global _
#totals section
lineToAdd = 'Total ( without groups )'
totals = calculateTotalsForEachColumn(lines)
splitFirstLine = lines[0].split(',')
for i in range(1, len(splitFirstLine)):
if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str(
splitFirstLine[i]).lower():
lineToAdd = lineToAdd + ',' + str(totals[i - 1])
else:
lineToAdd = lineToAdd + ','
lines.append(lineToAdd)
#means section
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
nbSourlients = getNbSourlients(lines, False)
#print "nbsourlients :%s" %nbSourlients
means = [0 for i in range(len(lines[0].split(',')) - 1)]
for i in range(len(totals)):
means[i] = float(totals[i]) / float(nbSourlients)
lines.append(
_('Means ( without groups ),') +
str(means).replace('[', '').replace(']', ''))
return lines
def __generateAllGraphicsForGroups(self, graphicType):
"""
@summary : Generated groups graphics based on the
specified graphicType.
@summary graphicType : "daily", "weekly", "monthly", "yearly"
@raise Exception: When graphicType is unknown.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
supportedGraphicTypes = {
"daily": "-d",
"weekly": "-w",
"monthly": "-m",
"yearly": "-y"
}
if graphicType not in supportedGraphicTypes:
raise Exception(
"Unsupported graphicType detected in __generateAllGraphicsForGroups"
)
else:
for group in configParameters.groupParameters.groups:
groupMembers, groupMachines, groupProducts, groupFileTypes = configParameters.groupParameters.getAssociatedParametersInStringFormat(
group)
groupMachines = str(groupMachines).replace("[", "").replace(
"]", "").replace("'", "").replace('"', '')
if graphicType == "daily":
commands.getstatusoutput(
'%sgenerateGnuGraphics.py -g %s -c %s --combineClients --copy -d "%s" -m %s -f %s -p %s -s 24 --outputLanguage %s'
% (self.paths.STATSBIN, group, groupMembers,
self.timeOfRequest, groupMachines, groupFileTypes,
groupProducts, self.outputLanguage))
#print '%sgenerateGnuGraphics.py -g %s -c %s --combineClients --fixedCurrent --copy -d "%s" -m %s -f %s -p %s -s 24 --language %s' %( self.paths.STATSBIN, group, groupMembers, self.timeOfRequest, groupMachines, groupFileTypes, groupProducts, self.outputLanguage )
else:
commands.getoutput(
"%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s"
% (self.paths.STATSBIN,
supportedGraphicTypes[graphicType], groupFileTypes,
groupMachines, group, self.timeOfRequest,
self.outputLanguage))
print "%sgenerateRRDGraphics.py %s --copy -f %s --machines '%s' -c %s --date '%s' --fixedCurrent --language %s" % (
self.paths.STATSBIN,
supportedGraphicTypes[graphicType], groupFileTypes,
groupMachines, group, self.timeOfRequest,
self.outputLanguage)
def addTotalsAndMeansToLines( lines ):
"""
@summary : Calculates the total of
every field found in the
spreadsheet and add the
line containing the totals
to the lines received as a
parameter.
@param lines: List of lines contained in
the spreadsheet we want to
calculate the totals for.
@return : Returns the lines with the
totals line appended to the
list of lines.
"""
global _
#totals section
lineToAdd = 'Total ( without groups )'
totals = calculateTotalsForEachColumn( lines )
splitFirstLine = lines[0].split(',')
for i in range(1, len( splitFirstLine ) ):
if _('total') in str(splitFirstLine[i]).lower() or _('cost') in str(splitFirstLine[i]).lower():
lineToAdd = lineToAdd + ',' + str(totals[i-1])
else:
lineToAdd = lineToAdd + ','
lines.append( lineToAdd )
#means section
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
nbSourlients = getNbSourlients( lines, False )
#print "nbsourlients :%s" %nbSourlients
means = [0 for i in range( len( lines[0].split(',') ) -1 ) ]
for i in range( len( totals ) ):
means[i] = float(totals[i]) / float( nbSourlients )
lines.append( _('Means ( without groups ),') + str(means).replace( '[', '' ).replace( ']', '' ) )
return lines
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Generates all the csv files required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
#costs
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
clusters = str(configParameters.sourceMachinesTags).replace(
'[', '').replace(']', '').replace(' ',
'').replace('"',
'').replace("'", "")
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, clusters, 0)
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts)
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllRRDGraphicsForWebPage( self, graphicType, generateTotalsGraphics = True ):
"""
@summary : This method generates new rrd graphics
based on the specified graphics
@param graphicType : daily weekly monthly or yearly
@raise Exception : When graphicType is unknown.
"""
supportedGraphicTypes = { "daily": "-d", "weekly":"-w", "monthly":"-m", "yearly":"-y" }
if graphicType not in supportedGraphicTypes:
raise Exception( "Unsupported graphicType detected in __generateAllGraphicsForGroups" )
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getListOfPairsAssociatedWithListOfTags(configParameters.sourceMachinesTags)
for machinePair in machinePairs:
machinePair = str(machinePair).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace( "'", "" ).replace( '"','' )
#individual graphics
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f tx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print "%sgenerateRRDGraphics.py %s --copy -f rx --machines '%s' --havingRun --date '%s' --fixedCurrent --language %s"\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
if generateTotalsGraphics == True :
#print output
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
# print '%sgenerateRRDGraphics.py %s --copy --totals -f "rx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage )
commands.getstatusoutput( '%sgenerateRRDGraphics.py %s --copy --totals -f "tx" --machines "%s" --havingRun --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, supportedGraphicTypes[graphicType], machinePair, self.timeOfRequest, self.outputLanguage ) )
def generateTopWebPage(self):
"""
@summary : Generates the top web page based on the
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
supportedLanguages = LanguageTools.getSupportedLanguages()
self.__createTheWebPage( configParameters.sourceMachinesTags, supportedLanguages )
def getMainApplicationLanguage():
"""
@summary : Reads and returns the main application
language form the config file.
@return : Le main application language.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
return configParameters.mainApplicationLanguage
def getMainApplicationLanguage():
"""
@summary : Reads and returns the main application
language form the config file.
@return : Le main application language.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
return configParameters.mainApplicationLanguage
def generateTopWebPage(self):
"""
@summary : Generates the top web page based on the
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
supportedLanguages = LanguageTools.getSupportedLanguages()
self.__createTheWebPage(configParameters.sourceMachinesTags,
supportedLanguages)
def updatePickleFiles(infos):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs(infos)
if needToupdateLogFiles == True:
updateLogFiles()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system("clear")
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f rx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" % (sourceMachine)
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f tx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" % (sourceMachine)
def updatePickleFiles( infos ):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs( infos )
if needToupdateLogFiles == True :
updateLogFiles()
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f rx -m %s "%( StatsPaths.STATSBIN, sourceMachine ) )
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" %(sourceMachine)
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN, sourceMachine) )
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" %(sourceMachine)
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(
self):
"""
@summary : Gets all the graphics required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
#-------------------- print "time of the request : ", self.timeOfRequest
# print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.dailyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForDailyWebPage(True, True, True)
# print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.weeklyWebPageFrequency](
self.timeOfRequest) == True:
#print "weeklies need to be updated."
self.generateAllForWeeklyWebPage(True, True)
# print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.monthlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForMonthlyWebPage(True, True)
# print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency
if requiresUpdateFonctions[
configParameters.timeParameters.yearlyWebPageFrequency](
self.timeOfRequest) == True:
self.generateAllForYearlyWebPage(True, True)
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() - (120 * 60)
endTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat))
startTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat -
(StatsDateLib.DAY * 7)))
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[groupName]
machines = statsConfig.groupParameters.groupsMachines[groupName]
fileType = statsConfig.groupParameters.groupFileTypes[groupName]
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=StatsDateLib.DAY * 7,
interval=StatsDateLib.HOUR)[:-1])
listOfFiles = PickleMerging.createMergedPicklesList(
startTime, endTime, clients, groupName, fileType, machines, seperators)
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles(
listOfFiles) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentFreeMemory, listOfFileSizes)
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" % (
currentFreeMemory, getTotalSizeListOfFiles(listOfFiles))
print "we have enough memory to merge all these pickles."
def main():
"""
@summary: Calls up the different methods required to set up
the interface.
"""
if len(sys.argv) == 2:
if sys.argv[1] == "-h" or sys.argv[1] == "--help":
printHelp()
else:
path = sys.argv[1]
currentlyUsedLanguages = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for languagePair in configParameters.webPagesLanguages:
if languagePair[0] not in currentlyUsedLanguages:
currentlyUsedLanguages.append(languagePair[0])
try:
if not isValidRootInstallationPath(path):
raise
createRootFolderIfNecessary(path)
copySourceFiles(currentlyUsedLanguages)
createSubFolders(path, currentlyUsedLanguages)
createSymbolicLinks(path, currentlyUsedLanguages)
giveOutPermissionsToFolders(currentlyUsedLanguages)
except:
print _(
"Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'."
)
sys.exit()
else:
print _(
"Error. Application must be called with one and only one parameter. Use -h|--help for further help."
)
def getGroups( fileType, machine):
"""
@summary : Gathers group found in config file.
@param fileType: Filetype for wich to search groups for.
@param machine : Machien for wich to search groups for.
@return: returns the list of groups matching the filetype and machine parameters
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine( fileType, machine )
return interestingGroups
def runPickleTransfersToRRDDatabases( infos ):
"""
@summary : Runs the transfer from pickles to rrd databases
from the start times found in the backup being used
and until the specified end time.
@param infos :
"""
os.system( "clear" )
showPresentation()
print ""
print "Updating databases...This may take a while..."
print ""
parameters = StatsConfigParameters( )
parameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in parameters.machinesToBackupInDb :
machines = machineParameters.getMachinesAssociatedWith(tag)
machines = str( machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) )
#print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime )
#print "output:%s" %output
print "Databases were updated for the following cluster : %s" %( tag )
if parameters.groupParameters.groups != []:
for group in parameters.groupParameters.groups :
groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupMachines = str( parameters.groupParameters.groupsMachines[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupProducts = str( parameters.groupParameters.groupsProducts[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupFileTypes = str(parameters.groupParameters.groupFileTypes[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) )
#print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts )
#print output
print "Databases were updated for the following group : %s " %( group )
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch(currentTime)
currentTime = StatsDateLib.getIsoWithRoundedSeconds(currentTime)
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len(sys.argv) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime(
timeToRestore,
'%Y-%m-%d %H:%M:%S') #will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" % (split[0], split[1])
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
restoreDatabases(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
else:
print _("You must specify a date.")
print _("Date must be of the folowing format YYYY-MM-DD HH:MM:SS")
print _("Program terminated.")
def getGroups(fileType, machine):
"""
@summary : Gathers group found in config file.
@param fileType: Filetype for wich to search groups for.
@param machine : Machien for wich to search groups for.
@return: returns the list of groups matching the filetype and machine parameters
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
interestingGroups = configParameters.groupParameters.getGroupsAssociatedWithFiletypeAndMachine(
fileType, machine)
return interestingGroups
def main():
"""
@summary: Calls up the different methods required to set up
the interface.
"""
if len( sys.argv ) == 2:
if sys.argv[1] == "-h" or sys.argv[1] == "--help":
printHelp()
else:
path = sys.argv[1]
currentlyUsedLanguages = []
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
for languagePair in configParameters.webPagesLanguages:
if languagePair[0] not in currentlyUsedLanguages:
currentlyUsedLanguages.append( languagePair[0] )
try:
if not isValidRootInstallationPath( path ) :
raise
createRootFolderIfNecessary( path )
copySourceFiles( currentlyUsedLanguages )
createSubFolders( path, currentlyUsedLanguages )
createSymbolicLinks( path, currentlyUsedLanguages )
giveOutPermissionsToFolders( currentlyUsedLanguages )
except :
print _("Specified folder must be an absolute path name. Please use folowing syntax : '/a/b/c/d'.")
sys.exit()
else:
print _("Error. Application must be called with one and only one parameter. Use -h|--help for further help.")
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentTime = StatsDateLib.getIsoWithRoundedSeconds( currentTime )
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len( sys.argv ) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime( timeToRestore, '%Y-%m-%d %H:%M:%S' )#will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" %( split[0], split[1] )
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
restoreDatabases( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
else:
print _( "You must specify a date." )
print _( "Date must be of the folowing format YYYY-MM-DD HH:MM:SS" )
print _( "Program terminated." )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig(self):
"""
@summary : Generates all the csv files required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
#costs
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","")
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour,"daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForDailyWebPage( True, clusters,0 )
if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts )
if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts )
if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def getCurrentlyActiveMachine():
"""
@return: Returns the list of currently active
source machines found within the config
file.
"""
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
currentlyActiveMachines=[]
for tag in configParameters.sourceMachinesTags:
currentlyActiveMachines.extend( machineParameters.getMachinesAssociatedWith(tag) )
currentlyActiveMachines.extend( [concat ( machineParameters.getMachinesAssociatedWith(tag)) for tag in configParameters.sourceMachinesTags ] )
return currentlyActiveMachines
def getNbSourlients( lines, includeGroups = False ):
"""
@summary : Goes through the received lines
and counts the number of sourlients
found.
@param lines :Lines to browse
@param includeGroups : Whether to include groups or not.
@return : Returns the number of sourlients found.
"""
global _
nbSourlients = 0
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
for line in lines :
entryIsValid = True
fields = line.split( ',' )
if includeGroups == False:
if str(fields[0].split( ' ' )[0]).replace(' ', '') in knownGroups :
entryIsValid = False
if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \
or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() :
entryIsValid = False
if entryIsValid == True:
nbSourlients = nbSourlients + 1
return nbSourlients
def generateAllForEverySupportedWebPages(self):
"""
@summary : Generates all the csv files required by
the web pages no matter what frequencies
are found within the config file.
"""
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
clusters = str( configParameters.sourceMachinesTags).replace('[', '').replace(']', '').replace(' ', '').replace('"','').replace("'","")
self.generateAllForDailyWebPage( True, clusters, 0 )
self.generateAllForWeeklyWebPage( True, clusters, weeklyCosts )
self.generateAllForMonthlyWebPage( True, clusters, monthlyCosts )
self.generateAllForYearlyWebPage( True, clusters, yearlyCosts )
def generateAllForEverySupportedWebPagesBasedOnFrequenciesFoundInConfig( self ):
"""
@summary : Gets all the graphics required by
the web pages based on the update
frequencies found within the config file.
@note: Supposes that the web pages
will require graphics from all the machines
specified in the configuration file.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
requiresUpdateFonctions = { "hourly": updateManager.isFirstUpdateOfTheHour, "daily": updateManager.isFirstUpdateOfTheDay, "weekly": updateManager.isFirstUpdateOfTheWeek,\
"monthly": updateManager.isFirstUpdateOfTheMonth, "yearly": updateManager.isFirstUpdateOfTheYear
}
#-------------------- print "time of the request : ", self.timeOfRequest
# print "daily frequency : ", configParameters.timeParameters.dailyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.dailyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForDailyWebPage( True, True, True )
# print "weekly frequency : ", configParameters.timeParameters.weeklyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.weeklyWebPageFrequency ](self.timeOfRequest) == True :
#print "weeklies need to be updated."
self.generateAllForWeeklyWebPage( True, True )
# print "montlhly frequency : ", configParameters.timeParameters.monthlyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.monthlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForMonthlyWebPage( True, True )
# print "yearly frequency : ", configParameters.timeParameters.yearlyWebPageFrequency
if requiresUpdateFonctions[ configParameters.timeParameters.yearlyWebPageFrequency ](self.timeOfRequest) == True :
self.generateAllForYearlyWebPage( True, True )
def updateWebPages(generalParameters):
"""
@summary : Generates all the required web pages
based on the language parameters found within
the configuration files.
"""
paths = StatsPaths()
paths.setPaths()
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
otherLanguages = []
generatorsTypes = [
DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator,
MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator,
TotalsGraphicsWebPageGenerator
]
for languagePair in generalParameters.webPagesLanguages:
for generatorsType in generatorsTypes:
generator = generatorsType(languagePair[0], languagePair[1])
generator.generateWebPage()
topWebPageGenerator = TopWebPageGenerator(languagePair[0])
topWebPageGenerator.generateTopWebPage()
otherLanguages.append(languagePair[0])
try:
while (1):
otherLanguages.remove(generalParameters.mainApplicationLanguage)
except:
pass
bottomWebPageGenerator = BottomWebPageGenerator(
generalParameters.mainApplicationLanguage, otherLanguages)
bottomWebPageGenerator.printWebPage()
def getNbSourlients(lines, includeGroups=False):
"""
@summary : Goes through the received lines
and counts the number of sourlients
found.
@param lines :Lines to browse
@param includeGroups : Whether to include groups or not.
@return : Returns the number of sourlients found.
"""
global _
nbSourlients = 0
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
knownGroups = configParameters.groupParameters.groups
for line in lines:
entryIsValid = True
fields = line.split(',')
if includeGroups == False:
if str(fields[0].split(' ')[0]).replace(' ', '') in knownGroups:
entryIsValid = False
if _('client') in str(fields[0]).lower() or _('source') in str(fields[0]).lower() \
or _('total') in str(fields[0]).lower() or _('mean') in str(fields[0]).lower() :
entryIsValid = False
if entryIsValid == True:
nbSourlients = nbSourlients + 1
return nbSourlients
def __generateAllMissingYearlyCsvFilesSinceLasteUpdate( self, clusters, cost):
"""
@summary : Generates the monthly graphics that were not
generated between last update and timeOfRequest
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingYears = updateManager.getMissingYearsBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingYear in missingYears:
self.timeOfRequest = missingYear
self.__generateAllRRDGraphicsForWebPage( "yearly", True )
self.__generateAllGraphicsForGroups( "yearly" )
self.timeOfRequest = oldTimeOfRequest
def generateWebPage( self ):
"""
@summary :
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.printWebPage( configParameters.sourceMachinesTags, machineParameters )
def __generateAllMissingDailyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight( missingDay )
self.__generateAllForDailyWebPage( False, generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "daily" )
self.timeOfRequest = oldTimeOfRequest
def __generateAllMissingWeeklyGraphicsSinceLasteUpdate( self, generateTotalsGraphics ):
"""
@summary : Generates the weekly graphics that were not
generated between last update and timeOfRequest
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingWeeks = updateManager.getMissingWeeksBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingWeek in missingWeeks:
self.timeOfRequest = missingWeek
self.__generateAllRRDGraphicsForWebPage( "weekly", generateTotalsGraphics )
self.__generateAllGraphicsForGroups( "weekly" )
self.timeOfRequest = oldTimeOfRequest
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.sourceMachinesTags)
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.picklingMachines)
for sourceMachine, picklingMachine in map(None, individualSourceMachines,
individualPicklingMachines):
if picklingMachine == LOCAL_MACHINE: #pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG,
sourceMachine, userName)
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (
userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine)
output = commands.getoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine))
print output
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() -(120*60)
endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) )
startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) )
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[ groupName ]
machines = statsConfig.groupParameters.groupsMachines[ groupName ]
fileType = statsConfig.groupParameters.groupFileTypes[ groupName ]
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1])
listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators )
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes )
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) )
print "we have enough memory to merge all these pickles."
def getRxTxNamesCurrentlyRunningOnAllMachinesfoundInConfigfile():
"""
@summary : Reads the config file and returns
all the currently running rx and tx names
associated with any of the source machines
found within the config file.
@return: Returns the rxNames and the txNames found.
"""
rxNames = []
txNames = []
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
for tag in configParameters.sourceMachinesTags:
machine = configParameters.detailedParameters.sourceMachinesForTag[ tag ][0]
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine)
rxNames.extend( filter( lambda x: x not in rxNames, newRxNames ) )
txNames.extend( filter( lambda x: x not in txNames, newTxNames ) )
return rxNames, txNames
def __generateAllMissingDailyCsvFilesSinceLasteUpdate( self, clusters, cost ):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
"""
if clusters != [] and clusters != None:
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
missingDays = updateManager.getMissingDaysBetweenUpdates( updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest )
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays:
self.timeOfRequest = missingDay
self.__generateAllGraphicsForDailyWebPage( False, True )
self.timeOfRequest = oldTimeOfRequest
def updateWebPages( generalParameters ):
"""
@summary : Generates all the required web pages
based on the language parameters found within
the configuration files.
"""
paths = StatsPaths()
paths.setPaths()
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
otherLanguages = []
generatorsTypes = [ DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator, MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator, TotalsGraphicsWebPageGenerator ]
for languagePair in generalParameters.webPagesLanguages :
for generatorsType in generatorsTypes :
generator = generatorsType( languagePair[0], languagePair[1] )
generator.generateWebPage()
topWebPageGenerator = TopWebPageGenerator(languagePair[0])
topWebPageGenerator.generateTopWebPage()
otherLanguages.append( languagePair[0] )
try:
while(1):
otherLanguages.remove( generalParameters.mainApplicationLanguage )
except:
pass
bottomWebPageGenerator = BottomWebPageGenerator(generalParameters.mainApplicationLanguage, otherLanguages )
bottomWebPageGenerator.printWebPage()
def generateAllForEverySupportedWebPages(self):
"""
@summary : Generates all the csv files required by
the web pages no matter what frequencies
are found within the config file.
"""
yearlyCosts = TOTAL_YEARLY_OPERATIONAL_COSTS
monthlyCosts = yearlyCosts / 12.0
weeklyCosts = yearlyCosts / 52.0
#Get params from configuration files
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
clusters = str(configParameters.sourceMachinesTags).replace(
'[', '').replace(']', '').replace(' ',
'').replace('"',
'').replace("'", "")
self.generateAllForDailyWebPage(True, clusters, 0)
self.generateAllForWeeklyWebPage(True, clusters, weeklyCosts)
self.generateAllForMonthlyWebPage(True, clusters, monthlyCosts)
self.generateAllForYearlyWebPage(True, clusters, yearlyCosts)
def __generateAllForDailyWebPage( self, copyToColumbosFolder = True,
generateTotalsGraphics = True ):
"""
@summary : Gets all the required daily graphs.
@param getGraphicsMissingSinceLastUpdate : Whether or not to generate
the daily graphics that did
not get generated since the
last update.
@param generateTotalsGraphics : Whether or not to generate the graphics
displaying the totals for each clusters.
@todo : Add proper support for copyToColumbosFolder
when generateAllGraphics finally support
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(configParameters.sourceMachinesTags)
for machineTag in configParameters.sourceMachinesTags:
logins = []
machines = configParameters.detailedParameters.sourceMachinesForTag[machineTag]
for machine in machines:
logins.append( machineConfig.getUserNameForMachine(machine) )
logins = str(logins).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
machines = str(machines).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
if "," in machines :
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) )
#print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
#%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
else:
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s "
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) )
#print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
if generateTotalsGraphics == True :
for machinePair in machinePairs:
#Generate all the daily total graphs.
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) )
#print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage)
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def __generateAllForDailyWebPage(self,
copyToColumbosFolder=True,
generateTotalsGraphics=True):
"""
@summary : Gets all the required daily graphs.
@param getGraphicsMissingSinceLastUpdate : Whether or not to generate
the daily graphics that did
not get generated since the
last update.
@param generateTotalsGraphics : Whether or not to generate the graphics
displaying the totals for each clusters.
@todo : Add proper support for copyToColumbosFolder
when generateAllGraphics finally support
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(
configParameters.sourceMachinesTags)
for machineTag in configParameters.sourceMachinesTags:
logins = []
machines = configParameters.detailedParameters.sourceMachinesForTag[
machineTag]
for machine in machines:
logins.append(machineConfig.getUserNameForMachine(machine))
logins = str(logins).replace("[", "").replace("]",
"").replace(" ", "")
machines = str(machines).replace("[",
"").replace("]",
"").replace(" ", "")
if "," in machines:
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) )
#print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
#%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
else:
output = commands.getoutput(
"%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s "
% (self.paths.STATSBIN, machines.replace(
"'", ""), logins.replace(
"'", ""), self.timeOfRequest, self.outputLanguage))
#print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
if generateTotalsGraphics == True:
for machinePair in machinePairs:
#Generate all the daily total graphs.
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) )
#print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage)
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def getParametersFromForm(self, form):
"""
@summary: Initialises the queryParameters
based on the form received as
parameter.
@note : Absent parameters will be set to default values( [] or '' )
and will NOT raise exceptions. Use the searchForParameterErrors
function to search for errors
"""
global _
#print form
image = None #No image was produced yet
#Every param is received in an array, use [0] to get first item, nothing for array.
try:
querier = form["querier"].replace("'", "").replace('"', '')
except:
querier = ''
try:
plotter = form["plotter"].replace("'", "").replace('"', '')
except:
plotter = ''
try:
fileTypes = form["fileType"].replace("'", "").replace('"', '')
except:
fileTypes = ''
try:
sourLients = form["sourLients"].split(',')
except:
sourLients = []
try:
groupName = form["groupName"].replace("'", "").replace('"', '')
except:
groupName = ''
try:
machines = form["machines"].split(',')
except:
machines = []
if groupName != '' and (sourLients == [] or sourLients == ['']):
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
if groupName in configParameters.groupParameters.groups:
if configParameters.groupParameters.groupFileTypes[
groupName] == fileTypes and configParameters.groupParameters.groupsMachines[
groupName] == machines:
sourLients = configParameters.groupParameters.groupsMembers[
groupName]
try:
combine = form["combineSourlients"].replace(",",
"").replace('"', '')
if combine == 'false' or combine == 'False':
combine = False
elif combine == 'true' or combine == 'True':
combine = True
else:
raise
except:
combine = False
try:
endTime = form["endTime"].replace("'", "").replace('"', '')
hour = endTime.split(" ")[1]
splitDate = endTime.split(" ")[0].split('-')
endTime = "%s" % (splitDate[2] + '-' + splitDate[1] + '-' +
splitDate[0] + " " + hour)
if _("current") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromCurrentDay(
endTime)
elif _("previous") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromPreviousDay(
endTime)
except:
endTime = ''
try:
products = form["products"].split(',')
if products == [""]:
raise
except:
products = ["*"]
try:
statsTypes = form["statsTypes"].split(',')
except:
statsTypes = []
#statsTypes = translateStatsTypes( statsTypes )
try:
span = form["span"].replace("'", "").replace('"', '')
if str(span).replace(' ', '') == '':
raise
span = int(span)
except:
span = 24
try:
language = form["lang"].replace("'", "").replace('"', '')
except:
language = ""
sourLients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
endTime, span, sourLients, machines, [fileTypes])
self.queryParameters = GnuQueryBroker._QueryParameters(
fileTypes, sourLients, groupName, machines, combine, endTime,
products, statsTypes, span, language)
self.replyParameters = GnuQueryBroker._ReplyParameters(
querier, plotter, image, fileTypes, sourLients, groupName,
machines, combine, endTime, products, statsTypes, span, '',
language)
def runPickleTransfersToRRDDatabases(infos):
"""
@summary : Runs the transfer from pickles to rrd databases
from the start times found in the backup being used
and until the specified end time.
@param infos :
"""
os.system("clear")
showPresentation()
print ""
print "Updating databases...This may take a while..."
print ""
parameters = StatsConfigParameters()
parameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in parameters.machinesToBackupInDb:
machines = machineParameters.getMachinesAssociatedWith(tag)
machines = str(machines).replace("[", "").replace("]",
"").replace(" ", "")
status, output = commands.getstatusoutput(
"%stransferPickleToRRD.py -m '%s' -e '%s'" %
(StatsPaths.STATSBIN, machines,
infos.databasesRecollectionEndTime))
#print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime )
#print "output:%s" %output
print "Databases were updated for the following cluster : %s" % (tag)
if parameters.groupParameters.groups != []:
for group in parameters.groupParameters.groups:
groupMembers = str(
parameters.groupParameters.groupsMembers[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupMachines = str(
parameters.groupParameters.groupsMachines[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupProducts = str(
parameters.groupParameters.groupsProducts[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupFileTypes = str(
parameters.groupParameters.groupFileTypes[group]).replace(
"[", "").replace("]", "").replace(" ", "")
status, output = commands.getstatusoutput(
"%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' "
% (StatsPaths.STATSBIN, groupMembers, groupMachines,
infos.databasesRecollectionEndTime, group, groupFileTypes,
groupProducts))
#print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts )
#print output
print "Databases were updated for the following group : %s " % (
group)
def printWebPage(self, rxNames, txNames):
"""
@summary : Generates a web page based on all the
rxnames and tx names that have run during
the past x years.
@param rxNames: List of sources for which to write
links to their yearly graphics.
@param txNames: List of clients for which to write
links to their yearly graphics.
@precondition: global _ translator must be set prior to calling this function.
@notes : Only links to available graphics will be
displayed.
@return : None
"""
global _
rxNamesArray = rxNames.keys()
txNamesArray = txNames.keys()
rxNamesArray.sort()
txNamesArray.sort()
# Redirect output towards html page to generate.
if not os.path.isdir(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML):
os.makedirs(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML)
fileHandle = open(
"%syearlyGraphs_%s.html" % (self.pathsTowardsOutputFiles.STATSWEBPAGESHTML, self.displayedLanguage), "w"
)
fileHandle.write(
"""
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<link rel="stylesheet" href="../scripts/js_%s/windowfiles/dhtmlwindow.css" type="text/css" />
<script type="text/javascript" src="../scripts/js_%s/windowfiles/dhtmlwindow.js">
This is left here to give credit to the original
creators of the dhtml script used for the group pop ups:
/***********************************************
* DHTML Window Widget- Dynamic Drive (www.dynamicdrive.com)
* This notice must stay intact for legal use.
* Visit http://www.dynamicdrive.com/ for full source code
***********************************************/
</script>
"""
% (self.displayedLanguage, self.displayedLanguage)
+ """
<script type="text/javascript">
var descriptionWindow=dhtmlwindow.open("description", "inline", "description", "Description", "width=900px,height=120px,left=150px,top=10px,resize=1,scrolling=0", "recal")
descriptionWindow.hide()
</script>
<head>
<title> PX Graphics </title>
<script>
counter =0;
function wopen(url, name, w, h){
// This function was taken on www.boutell.com
w += 32;
h += 96;
counter +=1;
var win = window.open(url,
counter,
'width=' + w + ', height=' + h + ', ' +
'location=no, menubar=no, ' +
'status=no, toolbar=no, scrollbars=no, resizable=no');
win.resizeTo(w, h);
win.focus();
}
</script>
<script>
function showSourceHelpPage(){
var sourceHelpPage = dhtmlwindow.open("sourceHelpPage", "iframe", "helpPages/source_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'source'")
+ """", "width=875px,height=100px,resize=1,scrolling=1,center=1", "recal")
sourceHelpPage.moveTo("middle", "middle");
}
function showBytecountHelpPage(){
var byteCountHelpPage = dhtmlwindow.open("byteCount", "iframe", "helpPages/byteCount_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'byteCount'")
+ """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
byteCountHelpPage.moveTo("middle", "middle");
}
function showClientHelpPage(){
var clientHelpPage = dhtmlwindow.open("client", "iframe", "helpPages/client_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'client'")
+ """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
.moveTo("middle", "middle");
}
function showErrorsHelpPage(){
var errorsHelpPage = dhtmlwindow.open("errors", "iframe", "helpPages/errors_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'errors'")
+ """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
errorsHelpPage.moveTo("middle", "middle");
}
function showFilecountHelpPage(){
var fileCountHelpPage = dhtmlwindow.open("fileCount", "iframe", "helpPages/fileCount_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'filecount'")
+ """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
fileCountHelpPage.moveTo("middle", "middle");
}
function showFilesOverMaxLatencyHelpPage(){
var filesOverMaxLatencyHelpPage = dhtmlwindow.open("filesOverMaxLatency", "iframe", "helpPages/filesOverMaxLatency_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'filesOverMaxLatency'")
+ """", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
filesOverMaxLatencyHelpPage.moveTo("middle", "middle");
}
function showLatencyHelpPage(){
var latencyHelpPage = dhtmlwindow.open("latency", "iframe", "helpPages/latency_%s.html" """
% self.displayedLanguage
+ """, " """
+ _("Definition of 'latency'")
+ """" , "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
latencyHelpPage.moveTo("middle", "middle");
}
</script>
<STYLE>
<!--
A{text-decoration:none}
-->
</STYLE>
<style type="text/css">
div.left { float: left; }
div.right {float: right; }
</style>
<style type="text/css">
a.blackLinks{
color: #000000;
}
div.tableContainer {
width: 95%; /* table width will be 99% of this*/
height: 275px; /* must be greater than tbody*/
overflow: auto;
margin: 0 auto;
}
table.cssTable {
width: 99%; /*100% of container produces horiz. scroll in Mozilla*/
border: none;
background-color: #f7f7f7;
table-layout: fixed;
}
table.cssTable>tbody { /* child selector syntax which IE6 and older do not support*/
overflow: auto;
height: 225px;
overflow-x: hidden;
}
thead tr {
position:relative;
}
thead td, thead th {
text-align: center;
font-size: 14px;
background-color:"#006699";
color: steelblue;
font-weight: bold;
border-top: solid 1px #d8d8d8;
}
td.cssTable {
color: #000;
padding-right: 2px;
font-size: 12px;
text-align: left;
border-bottom: solid 1px #d8d8d8;
border-left: solid 1px #d8d8d8;
}
tfoot td {
text-align: center;
font-size: 11px;
font-weight: bold;
background-color: papayawhip;
color: steelblue;
border-top: solid 2px slategray;
}
td:last-child {padding-right: 20px;} /*prevent Mozilla scrollbar from hiding cell content*/
</style>
</head>
<body text="#000000" link="#FFFFFF" vlink="000000" bgcolor="#FFF4E5" >
<br>
<table width = "100%">
<tr width = "100%">
<div class="left"><b><font size="5">"""
+ _("Yearly graphics for RX sources from MetPx.")
+ """ </font><font size = "2">"""
+ _("*updated monthly")
+ """</font></b></div>
"""
)
oneFileFound = False
for year in self.years:
parameters = StatsConfigParameters()
parameters.getAllParameters()
machinesStr = (
str(parameters.sourceMachinesTags)
.replace("[", "")
.replace("]", "")
.replace(",", "")
.replace("'", "")
.replace('"', "")
.replace(" ", "")
)
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = self.pathsTowardsGraphics.STATSCSVFILES + _("yearly/rx/%s/%s.csv") % (machinesStr, currentYear)
webLink = _("csvFiles/yearly/rx/%s/%s.csv") % (machinesStr, currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
if oneFileFound == False:
fileHandle.write(
"<div class='right'><font size='2' color='black'" + _("CSV files") + " : "
)
oneFileFound = True
fileHandle.write("""<a href="%s" class="blackLinks">%.4s.csv </a>""" % (webLink, currentYear))
if oneFileFound == True:
fileHandle.write(
"""
</font>
</div>
"""
)
fileHandle.write(
"""
</tr>
</td>
</table>
<br>
<br>
<div class="tableContainer">
<table class="cssTable">
<thead>
<tr>
<td bgcolor="#006699" class="cssTable">
<font color = "white">
<center>
"""
+ _("Sources")
+ """
<br>
<a target ="popup" href="#" onClick="showSourceHelpPage(); return false;">
?
</a>
<center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _("Display the total of bytes received every day of the week for each sources.")
+ """ ">
<font color = "white">
<center>
"""
+ _("Bytecount")
+ """
<br>
<a target ="popup" href="#" onClick="showBytecountHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _("Display the total of files received every day of the week for each sources.")
+ """ ">
<font color = "white">
<center>
"""
+ _("Filecount")
+ """
<br>
<a target ="popup" href="#" onClick="showFilecountHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title = " """
+ _(
"Display the total of errors that occured during the receptions for every day of the week for each sources."
)
+ """ ">
<font color = "white">
<center>
"""
+ _("Errors")
+ """
<br>
<a target ="popup" href="#" onClick="showErrorsHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
</tr>
</thead>
<tbody>
"""
)
for rxName in rxNamesArray:
if rxNames[rxName] == "":
fileHandle.write("""<tr> <td bgcolor="#99FF99" class="cssTable"> %s </td> """ % (rxName))
fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
else:
machineName = self.getMachineNameFromDescription(rxNames[rxName])
fileHandle.write(
"""<tr> <td bgcolor="#99FF99" class="cssTable"><div class="left"> %s </div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """
% (rxName, rxNames[rxName].replace("'", "").replace('"', ""), machineName)
)
fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/rx/%s/") % (rxName)
+ _("bytecount/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/rx/%s/") % (rxName) + _("bytecount/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (rxName, webLink, time.strftime("%Y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/rx/%s/") % (rxName)
+ _("filecount/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/rx/%s/") % (rxName) + _("filecount/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (rxName, webLink, time.strftime("%Y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/rx/%s/") % (rxName)
+ _("errors/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/rx/%s/") % (rxName) + _("errors/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (rxName, file, time.strftime("%Y", time.gmtime(year)))
)
fileHandle.write("</td></tr>")
fileHandle.write(
"""
</tbody>
</table>
</div>
<br>
<table width = "100%">
<tr width = "100%">
<div class="left"><b><font size="5">"""
+ _("Yearly graphics for TX clients from MetPx.")
+ """ </font><font size = "2">"""
+ _("*updated monthly")
+ """</font></b></div>
"""
)
oneFileFound = False
for year in self.years:
parameters = StatsConfigParameters()
parameters.getAllParameters()
machinesStr = (
str(parameters.sourceMachinesTags)
.replace("[", "")
.replace("]", "")
.replace(",", "")
.replace("'", "")
.replace('"', "")
.replace(" ", "")
)
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = self.pathsTowardsGraphics.STATSCSVFILES + _("yearly/tx/%s/%s.csv") % (machinesStr, currentYear)
webLink = _("csvFiles/yearly/tx/%s/%s.csv") % (machinesStr, currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
print file
if os.path.isfile(file):
if oneFileFound == False:
fileHandle.write(
"<div class='right'><font size='2' color='black'>" + _("CSV files") + " : "
)
oneFileFound = True
fileHandle.write("""<a href="%s" class="blackLinks">%.4s.csv </a>""" % (webLink, currentYear))
if oneFileFound == True:
fileHandle.write(
"""
</font>
</div>
"""
)
fileHandle.write(
"""
</tr>
</td>
</table>
<br>
<br>
<div class="tableContainer">
<table class="cssTable" >
<thead>
<tr>
<td bgcolor="#006699" class="cssTable">
<font color = "white">
<center>
"""
+ _("Clients")
+ """
<br>
<a target ="popup" href="#" onClick="showClientHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _("Display the taverage latency of file transfers for every day of the week for each clients.")
+ """>
<font color = "white">
<center>
"""
+ _("Latency")
+ """
<br>
<a target ="popup" href="#" onClick="showLatencyHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _(
"Display the total number of files for wich the latency was over 15 seconds for every day of the week for each clients."
)
+ """>
<font color = "white">
<center>
"""
+ _("Files Over Max. Lat.")
+ """
<br>
<a target ="popup" href="#" onClick="showFilesOverMaxLatencyHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _("Display the total of bytes transfered every day of the week for each clients.")
+ """>
<font color = "white">
<center>
"""
+ _("Bytecount")
+ """
<br>
<a target ="popup" href="#" onClick="showBytecountHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _("Display the total of files transferred every day of the week for each clients.")
+ """>
<font color = "white">
<center>
"""
+ _("Filecount")
+ """
<br>
<a target ="popup" href="#" onClick="showFilecountHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
<td bgcolor="#006699" class="cssTable" title =" """
+ _(
"Display the total of errors that occured during file transfers every day of the week for each clients."
)
+ """>
<font color = "white">
<center>
"""
+ _("Errors")
+ """
<br>
<a target ="popup" href="#" onClick="showErrorsHelpPage(); return false;">
?
</a>
</center>
</font>
</td>
</tr>
</thead>
<tbody>
"""
)
for txName in txNamesArray:
if txNames[txName] == "":
fileHandle.write(""" <tr> <td bgcolor="#99FF99" class="cssTable">%s</td> """ % (txName))
fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
else:
machineName = self.getMachineNameFromDescription(txNames[txName])
fileHandle.write(
"""<tr> <td bgcolor="#99FF99" class="cssTable"><div class="left"> %s </div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """
% (txName, txNames[txName].replace("'", "").replace('"', ""), machineName)
)
fileHandle.write("""<td bgcolor="#66CCFF" class="cssTable">""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/tx/%s/") % (txName)
+ _("latency/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/tx/%s/") % (txName) + _("latency/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (txName, webLink, time.strftime("%y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/tx/%s/") % (txName)
+ _("filesOverMaxLatency/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/tx/%s/") % (txName) + _("filesOverMaxLatency/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (txName, webLink, time.strftime("%y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/tx/%s/") % (txName)
+ _("bytecount/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/tx/%s/") % (txName) + _("bytecount/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (txName, webLink, time.strftime("%y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF">""" + _("Years") + """ : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/tx/%s/") % (txName)
+ _("filecount/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/tx/%s/") % (txName) + _("filecount/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (txName, webLink, time.strftime("%y", time.gmtime(year)))
)
fileHandle.write("</td>")
fileHandle.write(""" <td bgcolor="#66CCFF" class="cssTable" >Years : """)
for year in self.years:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(year)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.filesLanguage)
file = (
self.pathsTowardsGraphics.STATSGRAPHSARCHIVES
+ _("yearly/tx/%s/") % (txName)
+ _("errors/%s.png") % str(currentYear)
)
webLink = _("archives/yearly/tx/%s/") % (txName) + _("errors/%s.png") % str(currentYear)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write(
"""<a target ="popup" href="%s" onClick="wopen('%s', 'popup', 875, 240); return false;">%s </a>"""
% (txName, webLink, time.strftime("%y", time.gmtime(year)))
)
fileHandle.write("</td></tr>")
fileHandle.write(
"""
</tbody>
</table>
</div>
</body>
</html>
"""
)
fileHandle.close()
