def getDetailedParametersFromMachineConfig( self ):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map( None, self.sourceMachinesTags, self.picklingMachines ):
sourceMachines = machineParameters.getMachinesAssociatedWith( machineTag )
picklingMachines = machineParameters.getMachinesAssociatedWith( picklingMachine )
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] =[]
for machine in sourceMachines :
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(machine)
self.detailedParameters.sourceMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[machineTag]:
self.detailedParameters.picklingMachines[machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith( uploadMachine )
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines !=[]:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(machine)
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system("clear")
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(
3): #do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(machineParameters.getUserNameForMachine(sourceMachine),
sourceMachine, StatsPaths.PXLOG, StatsPaths.STATSLOGS,
sourceMachine))
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep(10)
def getRxTxNames(localMachine, machine):
"""
Returns a tuple containing RXnames and TXnames
of the currently running sources/clients of a
desired machine.
"""
pxManager = PXManager()
PXPaths.RX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles(
localMachine, machine, 'rx')
PXPaths.TX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles(
localMachine, machine, 'tx')
PXPaths.TRX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles(
localMachine, machine, 'trx')
pxManager.initNames() # Now you must call this method
if localMachine != machine:
try:
parameters = MachineConfigParameters()
parameters.getParametersFromMachineConfigurationFile()
userName = parameters.getUserNameForMachine(machine)
except:
userName = "******"
GeneralStatsLibraryMethods.updateConfigurationFiles(
machine, userName)
txNames = pxManager.getTxNames()
rxNames = pxManager.getRxNames()
return rxNames, txNames
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags )
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines )
for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) :
if picklingMachine == LOCAL_MACHINE :#pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName )
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine )
output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) )
print output
def getRxTxNames( localMachine, machine ):
"""
Returns a tuple containing RXnames and TXnames
of the currently running sources/clients of a
desired machine.
"""
pxManager = PXManager()
PXPaths.RX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles( localMachine, machine, 'rx' )
PXPaths.TX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles( localMachine, machine, 'tx' )
PXPaths.TRX_CONF = GeneralStatsLibraryMethods.getPathToConfigFiles( localMachine, machine, 'trx' )
pxManager.initNames() # Now you must call this method
if localMachine != machine :
try:
parameters = MachineConfigParameters()
parameters.getParametersFromMachineConfigurationFile()
userName = parameters.getUserNameForMachine(machine)
except:
userName = "******"
GeneralStatsLibraryMethods.updateConfigurationFiles( machine, userName )
txNames = pxManager.getTxNames()
rxNames = pxManager.getRxNames()
return rxNames, txNames
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.sourceMachinesTags)
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.picklingMachines)
for sourceMachine, picklingMachine in map(None, individualSourceMachines,
individualPicklingMachines):
if picklingMachine == LOCAL_MACHINE: #pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG,
sourceMachine, userName)
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (
userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine)
output = commands.getoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine))
print output
def __generateAllForDailyWebPage( self, copyToColumbosFolder = True,
generateTotalsGraphics = True ):
"""
@summary : Gets all the required daily graphs.
@param getGraphicsMissingSinceLastUpdate : Whether or not to generate
the daily graphics that did
not get generated since the
last update.
@param generateTotalsGraphics : Whether or not to generate the graphics
displaying the totals for each clusters.
@todo : Add proper support for copyToColumbosFolder
when generateAllGraphics finally support
"""
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(configParameters.sourceMachinesTags)
for machineTag in configParameters.sourceMachinesTags:
logins = []
machines = configParameters.detailedParameters.sourceMachinesForTag[machineTag]
for machine in machines:
logins.append( machineConfig.getUserNameForMachine(machine) )
logins = str(logins).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
machines = str(machines).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
if "," in machines :
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) )
#print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
#%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
else:
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s "
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage ) )
#print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
if generateTotalsGraphics == True :
for machinePair in machinePairs:
#Generate all the daily total graphs.
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) )
#print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage)
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def __generateAllForDailyWebPage(self,
copyToColumbosFolder=True,
generateTotalsGraphics=True):
"""
@summary : Gets all the required daily graphs.
@param getGraphicsMissingSinceLastUpdate : Whether or not to generate
the daily graphics that did
not get generated since the
last update.
@param generateTotalsGraphics : Whether or not to generate the graphics
displaying the totals for each clusters.
@todo : Add proper support for copyToColumbosFolder
when generateAllGraphics finally support
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinePairs = machineConfig.getPairedMachinesAssociatedWithListOfTags(
configParameters.sourceMachinesTags)
for machineTag in configParameters.sourceMachinesTags:
logins = []
machines = configParameters.detailedParameters.sourceMachinesForTag[
machineTag]
for machine in machines:
logins.append(machineConfig.getUserNameForMachine(machine))
logins = str(logins).replace("[", "").replace("]",
"").replace(" ", "")
machines = str(machines).replace("[",
"").replace("]",
"").replace(" ", "")
if "," in machines:
output = commands.getoutput( "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage) )
#print "%sgenerateAllGnuGraphicsForMachines.py -m '%s' -c -l '%s' --date '%s' --outputLanguage %s "\
#%( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
else:
output = commands.getoutput(
"%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s "
% (self.paths.STATSBIN, machines.replace(
"'", ""), logins.replace(
"'", ""), self.timeOfRequest, self.outputLanguage))
#print "%sgenerateAllGnuGraphicsForMachines.py -i -m '%s' -l '%s' --date '%s' --outputLanguage %s " %( self.paths.STATSBIN, machines.replace( "'","" ), logins.replace( "'","" ), self.timeOfRequest, self.outputLanguage )
#print output
if generateTotalsGraphics == True:
for machinePair in machinePairs:
#Generate all the daily total graphs.
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage) )
#print '%sgenerateRRDGraphics.py --copy --totals -f "rx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
# %( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage)
commands.getoutput( '%sgenerateRRDGraphics.py --copy --totals -f "tx" --machines "%s" -d --fixedCurrent --date "%s" --language %s'\
%( self.paths.STATSBIN, machinePair, self.timeOfRequest, self.outputLanguage ) )
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(3):#do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) )
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep( 10 )
def getDetailedParametersFromMachineConfig(self):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map(None, self.sourceMachinesTags,
self.picklingMachines):
sourceMachines = machineParameters.getMachinesAssociatedWith(
machineTag)
picklingMachines = machineParameters.getMachinesAssociatedWith(
picklingMachine)
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] = []
for machine in sourceMachines:
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[
machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(
machine)
self.detailedParameters.sourceMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[
machineTag]:
self.detailedParameters.picklingMachines[
machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith(
uploadMachine)
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines != []:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(
machine)
