def getDetailedParametersFromMachineConfig( self ):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map( None, self.sourceMachinesTags, self.picklingMachines ):
sourceMachines = machineParameters.getMachinesAssociatedWith( machineTag )
picklingMachines = machineParameters.getMachinesAssociatedWith( picklingMachine )
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] =[]
for machine in sourceMachines :
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(machine)
self.detailedParameters.sourceMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[machineTag]:
self.detailedParameters.picklingMachines[machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith( uploadMachine )
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines !=[]:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(machine)
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system("clear")
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(
3): #do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(machineParameters.getUserNameForMachine(sourceMachine),
sourceMachine, StatsPaths.PXLOG, StatsPaths.STATSLOGS,
sourceMachine))
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep(10)
def updateLogFiles():
"""
@summary : Downloads the log files from the source machines
into the local machine.
"""
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating log files...This may take a while...."
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
for i in range(3):#do 3 times in case of currently turning log files.
status, output = commands.getstatusoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine ) )
#print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachine ), sourceMachine , StatsPaths.PXLOG, StatsPaths.STATSLOGS, sourceMachine )
#print output
time.sleep( 10 )
def getCurrentlyActiveMachine():
"""
@return: Returns the list of currently active
source machines found within the config
file.
"""
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
currentlyActiveMachines=[]
for tag in configParameters.sourceMachinesTags:
currentlyActiveMachines.extend( machineParameters.getMachinesAssociatedWith(tag) )
currentlyActiveMachines.extend( [concat ( machineParameters.getMachinesAssociatedWith(tag)) for tag in configParameters.sourceMachinesTags ] )
return currentlyActiveMachines
def getGroupSettingsFromConfigurationFile( self ):
"""
Reads all the group settings from
the configuration file.
"""
groupParameters = GroupConfigParameters([], {}, {}, {},{} )
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
paths = StatsPaths()
paths.setBasicPaths()
config = paths.STATSETC + "config"
fileHandle = open( config, "r" )
line = fileHandle.readline()#read until groups section, or EOF
while line != "" and "[specialGroups]" not in line:
line = fileHandle.readline()
if line != "":#read until next section, or EOF
line = fileHandle.readline()
while line != "" and "[" not in line:
if line != '\n' and line[0] != '#' :
splitLine = line.split()
if len( splitLine ) == 6:
groupName = splitLine[0]
if groupName not in (groupParameters.groups):
groupParameters.groups.append( groupName )
groupParameters.groupsMachines[groupName] = []
groupParameters.groupFileTypes[groupName] = []
groupParameters.groupsMembers[groupName] = []
groupParameters.groupsProducts[groupName] = []
machines = splitLine[2].split(",")
for machine in machines:
groupParameters.groupsMachines[groupName].extend( machineParameters.getMachinesAssociatedWith(machine) )
groupParameters.groupFileTypes[groupName] = splitLine[3]
groupParameters.groupsMembers[groupName].extend( splitLine[4].split(",") )
groupParameters.groupsProducts[groupName].extend( splitLine[5].split(",") )
line = fileHandle.readline()
fileHandle.close()
self.groupParameters = groupParameters
def updateMachineNamesBasedOnExistingMachineTags(self):
"""
@summary : browses the list of existing machine
tags to see if the specified name was
a tag or not.
"""
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for i in range( len( self.machines ) ):
if self.machines[i] in machineParameters.getMachineTags():
self.machines[i] = str( machineParameters.getMachinesAssociatedWith(self.machines[i])).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
def updatePickleFiles(infos):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs(infos)
if needToupdateLogFiles == True:
updateLogFiles()
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system("clear")
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f rx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" % (sourceMachine)
status, output = commands.getstatusoutput(
"python %spickleUpdater.py -f tx -m %s " %
(StatsPaths.STATSBIN, sourceMachine))
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" % (sourceMachine)
def updatePickleFiles( infos ):
"""
@summary : Updates pickles files from the
specified start time to the specified
end time.
@param infos :
@note : If update is not up to now, we presume that
updating log files could cause us to loose
precious log files. Therefore we update log
files only if update is up to now, where we
absolutely need recent log files.
"""
needToupdateLogFiles = askUserAboutUpdatingLogs( infos )
if needToupdateLogFiles == True :
updateLogFiles()
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
os.system( "clear" )
showPresentation()
print ""
print ""
print "Updating pickles....This may take a while..."
print ""
for tag in configParameters.sourceMachinesTags:
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for sourceMachine in sourceMachines:
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f rx -m %s "%( StatsPaths.STATSBIN, sourceMachine ) )
#print output
#print "python %spickleUpdater.py -f rx -m %s " %( StatsPaths.STATSBIN, sourceMachine )
print "Updated rx pickles for : %s" %(sourceMachine)
status, output = commands.getstatusoutput( "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN, sourceMachine) )
#print "python %spickleUpdater.py -f tx -m %s " %( StatsPaths.STATSBIN,sourceMachine )
#print output
print "Updated tx pickles for : %s" %(sourceMachine)
def runPickleTransfersToRRDDatabases( infos ):
"""
@summary : Runs the transfer from pickles to rrd databases
from the start times found in the backup being used
and until the specified end time.
@param infos :
"""
os.system( "clear" )
showPresentation()
print ""
print "Updating databases...This may take a while..."
print ""
parameters = StatsConfigParameters( )
parameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in parameters.machinesToBackupInDb :
machines = machineParameters.getMachinesAssociatedWith(tag)
machines = str( machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime ) )
#print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime )
#print "output:%s" %output
print "Databases were updated for the following cluster : %s" %( tag )
if parameters.groupParameters.groups != []:
for group in parameters.groupParameters.groups :
groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupMachines = str( parameters.groupParameters.groupsMachines[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupProducts = str( parameters.groupParameters.groupsProducts[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
groupFileTypes = str(parameters.groupParameters.groupFileTypes[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" )
status, output = commands.getstatusoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts ) )
#print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts )
#print output
print "Databases were updated for the following group : %s " %( group )
def runPickleTransfersToRRDDatabases(infos):
"""
@summary : Runs the transfer from pickles to rrd databases
from the start times found in the backup being used
and until the specified end time.
@param infos :
"""
os.system("clear")
showPresentation()
print ""
print "Updating databases...This may take a while..."
print ""
parameters = StatsConfigParameters()
parameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for tag in parameters.machinesToBackupInDb:
machines = machineParameters.getMachinesAssociatedWith(tag)
machines = str(machines).replace("[", "").replace("]",
"").replace(" ", "")
status, output = commands.getstatusoutput(
"%stransferPickleToRRD.py -m '%s' -e '%s'" %
(StatsPaths.STATSBIN, machines,
infos.databasesRecollectionEndTime))
#print "%stransferPickleToRRD.py -m '%s' -e '%s'" %(StatsPaths.STATSBIN, machines, infos.databasesRecollectionEndTime )
#print "output:%s" %output
print "Databases were updated for the following cluster : %s" % (tag)
if parameters.groupParameters.groups != []:
for group in parameters.groupParameters.groups:
groupMembers = str(
parameters.groupParameters.groupsMembers[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupMachines = str(
parameters.groupParameters.groupsMachines[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupProducts = str(
parameters.groupParameters.groupsProducts[group]).replace(
"[", "").replace("]", "").replace(" ", "")
groupFileTypes = str(
parameters.groupParameters.groupFileTypes[group]).replace(
"[", "").replace("]", "").replace(" ", "")
status, output = commands.getstatusoutput(
"%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' "
% (StatsPaths.STATSBIN, groupMembers, groupMachines,
infos.databasesRecollectionEndTime, group, groupFileTypes,
groupProducts))
#print "%stransferPickleToRRD.py -c '%s' -m '%s' -e '%s' -g '%s' -f %s -p '%s' " %( StatsPaths.STATSBIN, groupMembers, groupMachines, infos.databasesRecollectionEndTime, group, groupFileTypes, groupProducts )
#print output
print "Databases were updated for the following group : %s " % (
group)
def getOptionsFromParser(parser):
"""
@summary: Parses and validates the options found in the parser.
@return: If information was found to be valid, return options
"""
infos = None
date = []
(options, args) = parser.parse_args()
machines = options.machines.replace(' ', '').split(',')
date = options.date.replace('"', '').replace("'", '')
fileType = options.fileType.replace("'", '')
daily = options.daily
weekly = options.weekly
monthly = options.monthly
yearly = options.yearly
fixedCurrent = options.fixedCurrent
fixedPrevious = options.fixedPrevious
turnOffLogging = options.turnOffLogging
includeGroups = options.includeGroups
machinesAreClusters = options.machinesAreClusters
outputLanguage = options.outputLanguage
if fixedPrevious and fixedCurrent:
print _(
"Error. Please use only one of the fixed options,either fixedPrevious or fixedCurrent. "
)
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
counter = 0
specialParameters = [daily, monthly, weekly, yearly]
for specialParameter in specialParameters:
if specialParameter:
counter = counter + 1
if counter > 1:
print _(
"Error. Only one of the daily, weekly and yearly options can be use at a time "
)
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif counter == 0:
print _("Error. Please use either the -d -m -w or -y options. ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime(date, '%Y-%m-%d %H:%M:%S')
split = date.split()
date = "%s %s" % (split[0], split[1])
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
#TODO :fixStartEnd method???
if fixedPrevious:
if daily:
span = "daily"
graphicType = "daily"
start, end = StatsDateLib.getStartEndFromPreviousDay(date)
elif weekly:
span = "weekly"
graphicType = "weekly"
start, end = StatsDateLib.getStartEndFromPreviousWeek(date)
elif monthly:
span = "monthly"
graphicType = "monthly"
start, end = StatsDateLib.getStartEndFromPreviousMonth(date)
elif yearly:
span = "yearly"
graphicType = "yearly"
start, end = StatsDateLib.getStartEndFromPreviousYear(date)
timeSpan = int(
StatsDateLib.getSecondsSinceEpoch(end) -
StatsDateLib.getSecondsSinceEpoch(start)) / 3600
elif fixedCurrent:
if daily:
span = "daily"
graphicType = "daily"
start, end = StatsDateLib.getStartEndFromCurrentDay(date)
elif weekly:
span = "weekly"
graphicType = "weekly"
start, end = StatsDateLib.getStartEndFromCurrentWeek(date)
elif monthly:
span = "monthly"
graphicType = "monthly"
start, end = StatsDateLib.getStartEndFromCurrentMonth(date)
elif yearly:
span = "yearly"
graphicType = "yearly"
start, end = StatsDateLib.getStartEndFromCurrentYear(date)
timeSpan = int(
StatsDateLib.getSecondsSinceEpoch(end) -
StatsDateLib.getSecondsSinceEpoch(start)) / 3600
else:
#TODO fix span method???
if daily:
timeSpan = 24
graphicType = "daily"
span = "daily"
elif weekly:
timeSpan = 24 * 7
graphicType = "weekly"
span = "weekly"
elif monthly:
timeSpan = 24 * 30
graphicType = "monthly"
span = "monthly"
elif yearly:
timeSpan = 24 * 365
graphicType = "yearly"
span = "yearly"
start = StatsDateLib.getIsoFromEpoch(
StatsDateLib.getSecondsSinceEpoch(date) - timeSpan * 60 * 60)
end = date
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if includeGroups == True:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
machinesToSearch = machines[:] #Forces a copy and nota reference.
for machine in machines:
if machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinesAssociatedWith = machineConfig.getMachinesAssociatedWith(
machine)
machinesToTest = str(machinesAssociatedWith).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).\
replace(" ",'').replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
for group in groups:
groupsMachine = str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
#print "machinesToTest %s groupsMachine %s" %(machinesToTest,groupsMachine )
if machinesToTest in groupsMachine:
if groupsMachine not in machinesToSearch:
machinesToSearch.append(groupsMachine)
#print machines
infos = _CsvInfos( start = start , end = end , span = span, timeSpan = timeSpan, fileType = fileType, machinesForLabels = machines,\
machinesToSearch = machinesToSearch, machinesAreClusters = machinesAreClusters, dataSource = "databases", outputLanguage = outputLanguage )
return infos
def getRxTxNamesForWebPages( start, end ):
"""
@summary: Returns two dictionaries, rx and tx, whose
keys is the list of rx or tx having run
between start and end.
If key has an associated value different from
"", this means that the entry is a group tag name.
The value will be an html description of the group.
@param start: Start of the span to look into.
@param end: End of the span to look into.
@return: see summary.
"""
rxNames = {}
txNames = {}
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for sourceMachinesTag in configParameters.sourceMachinesTags:
machines = machineParameters.getMachinesAssociatedWith( sourceMachinesTag )
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( start, end, machines )
for rxName in newRxNames :
description = "<font color='#008800'>--Source Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> " %(rxName, str(machines).replace('[', '').replace(']', '') )
rxNames[rxName] = description
for txName in newTxNames:
description = "<font color='#008800'>--Client Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> " %(txName, str(machines).replace('[', '').replace(']', '') )
txNames[txName] = description
for group in configParameters.groupParameters.groups:
#print group
machines = configParameters.groupParameters.groupsMachines[group]
machines = str(machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace(",",", ")
members = configParameters.groupParameters.groupsMembers[group]
members = str( members ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace(",",", ")
fileTypes = configParameters.groupParameters.groupFileTypes[group]
fileTypes = str(fileTypes ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace(",",", ")
products = configParameters.groupParameters.groupsProducts[group]
products = str( products ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ).replace(",",", ")
description = "<font color='#008800'>--Group Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--Member(s) : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--FileType : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--Product(s) pattern(s) : </font><font color='#006699'>%s</font> " %(group, machines, members, fileTypes, products )
if configParameters.groupParameters.groupFileTypes[group] == "tx":
txNames[group] = description
elif configParameters.groupParameters.groupFileTypes[group] == "rx":
rxNames[group] = description
return rxNames, txNames
def getDetailedParametersFromMachineConfig(self):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map(None, self.sourceMachinesTags,
self.picklingMachines):
sourceMachines = machineParameters.getMachinesAssociatedWith(
machineTag)
picklingMachines = machineParameters.getMachinesAssociatedWith(
picklingMachine)
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] = []
for machine in sourceMachines:
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[
machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(
machine)
self.detailedParameters.sourceMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[
machineTag]:
self.detailedParameters.picklingMachines[
machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith(
uploadMachine)
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines != []:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(
machine)
def getDataFromDatabases(sourlients, dataTypes, infos):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys():
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith(machine)
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes:
if infos.outputLanguage != 'en':
translatedDataType = LanguageTools.translateDataType(
dataType, "en", infos.outputLanguage)
else:
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[sourlient],
machines=[machine],
fileType=infos.fileType)
if not os.path.isfile(databaseName):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
groupName=sourlient,
machines=[machine],
fileType=infos.fileType,
usage="group")
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(
databaseName, infos.fileType)
fetchedInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="fetchData")
desiredInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="plotGraphic")
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def getOptionsFromParser( parser ):
"""
@summary: Parses and validates the options found in the parser.
@return: If information was found to be valid, return options
"""
infos = None
date = []
( options, args )= parser.parse_args()
machines = options.machines.replace( ' ','').split(',')
date = options.date.replace('"','').replace("'",'')
fileType = options.fileType.replace("'",'')
daily = options.daily
weekly = options.weekly
monthly = options.monthly
yearly = options.yearly
fixedCurrent = options.fixedCurrent
fixedPrevious = options.fixedPrevious
turnOffLogging = options.turnOffLogging
includeGroups = options.includeGroups
machinesAreClusters = options.machinesAreClusters
outputLanguage = options.outputLanguage
if fixedPrevious and fixedCurrent:
print _("Error. Please use only one of the fixed options,either fixedPrevious or fixedCurrent. " )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
counter = 0
specialParameters = [daily, monthly, weekly, yearly]
for specialParameter in specialParameters:
if specialParameter:
counter = counter + 1
if counter > 1 :
print _( "Error. Only one of the daily, weekly and yearly options can be use at a time " )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
elif counter == 0:
print _( "Error. Please use either the -d -m -w or -y options. " )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( date, '%Y-%m-%d %H:%M:%S' )
split = date.split()
date = "%s %s" %( split[0], split[1] )
except:
print _( "Error. The date format must be YYYY-MM-DD HH:MM:SS" )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
#TODO :fixStartEnd method???
if fixedPrevious :
if daily :
span = "daily"
graphicType = "daily"
start, end = StatsDateLib.getStartEndFromPreviousDay( date )
elif weekly:
span = "weekly"
graphicType = "weekly"
start, end = StatsDateLib.getStartEndFromPreviousWeek( date )
elif monthly:
span = "monthly"
graphicType = "monthly"
start, end = StatsDateLib.getStartEndFromPreviousMonth( date )
elif yearly:
span = "yearly"
graphicType = "yearly"
start, end = StatsDateLib.getStartEndFromPreviousYear( date )
timeSpan = int( StatsDateLib.getSecondsSinceEpoch( end ) - StatsDateLib.getSecondsSinceEpoch( start ) ) / 3600
elif fixedCurrent:
if daily :
span = "daily"
graphicType = "daily"
start, end = StatsDateLib.getStartEndFromCurrentDay( date )
elif weekly:
span = "weekly"
graphicType = "weekly"
start, end = StatsDateLib.getStartEndFromCurrentWeek( date )
elif monthly:
span = "monthly"
graphicType = "monthly"
start, end = StatsDateLib.getStartEndFromCurrentMonth( date )
elif yearly:
span = "yearly"
graphicType = "yearly"
start, end = StatsDateLib.getStartEndFromCurrentYear( date )
timeSpan = int( StatsDateLib.getSecondsSinceEpoch( end ) - StatsDateLib.getSecondsSinceEpoch( start ) ) / 3600
else:
#TODO fix span method???
if daily :
timeSpan = 24
graphicType = "daily"
span = "daily"
elif weekly:
timeSpan = 24 * 7
graphicType = "weekly"
span = "weekly"
elif monthly:
timeSpan = 24 * 30
graphicType = "monthly"
span = "monthly"
elif yearly:
timeSpan = 24 * 365
graphicType = "yearly"
span = "yearly"
start = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( date ) - timeSpan*60*60 )
end = date
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if includeGroups == True:
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
machinesToSearch = machines[:]#Forces a copy and nota reference.
for machine in machines:
if machinesAreClusters == True :
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machinesAssociatedWith = machineConfig.getMachinesAssociatedWith( machine )
machinesToTest = str(machinesAssociatedWith).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).\
replace(" ",'').replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
for group in groups:
groupsMachine = str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
#print "machinesToTest %s groupsMachine %s" %(machinesToTest,groupsMachine )
if machinesToTest in groupsMachine :
if groupsMachine not in machinesToSearch:
machinesToSearch.append(groupsMachine)
#print machines
infos = _CsvInfos( start = start , end = end , span = span, timeSpan = timeSpan, fileType = fileType, machinesForLabels = machines,\
machinesToSearch = machinesToSearch, machinesAreClusters = machinesAreClusters, dataSource = "databases", outputLanguage = outputLanguage )
return infos
def getDataFromDatabases( sourlients, dataTypes, infos ):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys() :
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch :
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith( machine )
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes :
if infos.outputLanguage != 'en' :
translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage )
else :
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients= [sourlient], machines = [machine], fileType = infos.fileType )
if not os.path.isfile( databaseName ):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(dataType = translatedDataType, groupName = sourlient, machines = [machine], fileType = infos.fileType, usage = "group" )
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType )
fetchedInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "fetchData" )
desiredInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "plotGraphic" )
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def getAllClientOrSourcesNamesFromMachines( infos ):
"""
@summary : Goes through all the machines and finds out
wich client or sources currently run on each
of those machines.
To make sure no confusion arrises if to clinets
or source have the same name on different
machhines or cluster, the returned names will
be associated with all the machines/clusters
with whom they are associated as to let the caller
hadnle the situation as it pleases.
@param infos: Infos that were gathered at program call.
@return : The dictionary containing the names and their associated machines.
"""
sourlients ={}
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith( machine )
if machines != []:
#print machines
machine = str( machines ).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, machines, pattern = None, havingrunOnAllMachines = True )
else:
rxNames, txNames = [],[]
else: #might be a groups machine,in this case do not force to have run on all machines.....
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, [machine], pattern = None, havingrunOnAllMachines = False )
else:
rxNames, txNames = [],[]
else:#not a cluster, une mahcine name directly. Force to have run exclusivly on specified machine.
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, [machine], pattern = None, havingrunOnAllMachines = True )
else:
rxNames, txNames = [],[]
if infos.fileType == "rx":
namesToAdd = rxNames
elif infos.fileType == "tx":
namesToAdd = txNames
for nameToAdd in namesToAdd:
if nameToAdd in sourlients.keys():
if machine not in sourlients[nameToAdd]:
sourlients[nameToAdd].append( machine )
else:
sourlients[nameToAdd] = [ machine ]
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
for group in groups :
if configParameters.groupParameters.groupFileTypes[group] == infos.fileType :
sourlients[group] = [str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')]
return sourlients
def getRxTxNamesForWebPages(start, end):
"""
@summary: Returns two dictionaries, rx and tx, whose
keys is the list of rx or tx having run
between start and end.
If key has an associated value different from
"", this means that the entry is a group tag name.
The value will be an html description of the group.
@param start: Start of the span to look into.
@param end: End of the span to look into.
@return: see summary.
"""
rxNames = {}
txNames = {}
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
for sourceMachinesTag in configParameters.sourceMachinesTags:
machines = machineParameters.getMachinesAssociatedWith(
sourceMachinesTag)
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
start, end, machines)
for rxName in newRxNames:
description = "<font color='#008800'>--Source Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> " % (
rxName, str(machines).replace('[', '').replace(']', ''))
rxNames[rxName] = description
for txName in newTxNames:
description = "<font color='#008800'>--Client Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> " % (
txName, str(machines).replace('[', '').replace(']', ''))
txNames[txName] = description
for group in configParameters.groupParameters.groups:
#print group
machines = configParameters.groupParameters.groupsMachines[group]
machines = str(machines).replace("[", "").replace("]", "").replace(
" ", "").replace(",", ", ")
members = configParameters.groupParameters.groupsMembers[group]
members = str(members).replace("[", "").replace("]", "").replace(
" ", "").replace(",", ", ")
fileTypes = configParameters.groupParameters.groupFileTypes[group]
fileTypes = str(fileTypes).replace("[",
"").replace("]", "").replace(
" ", "").replace(",", ", ")
products = configParameters.groupParameters.groupsProducts[group]
products = str(products).replace("[", "").replace("]", "").replace(
" ", "").replace(",", ", ")
description = "<font color='#008800'>--Group Name : </font> <font color='#006699'>%s</font> <br> <font color='#008800'>--Machine(s) : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--Member(s) : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--FileType : </font><font color='#006699'>%s</font> <br> <font color='#008800'>--Product(s) pattern(s) : </font><font color='#006699'>%s</font> " % (
group, machines, members, fileTypes, products)
if configParameters.groupParameters.groupFileTypes[group] == "tx":
txNames[group] = description
elif configParameters.groupParameters.groupFileTypes[
group] == "rx":
rxNames[group] = description
return rxNames, txNames
def getAllClientOrSourcesNamesFromMachines(infos):
"""
@summary : Goes through all the machines and finds out
wich client or sources currently run on each
of those machines.
To make sure no confusion arrises if to clinets
or source have the same name on different
machhines or cluster, the returned names will
be associated with all the machines/clusters
with whom they are associated as to let the caller
hadnle the situation as it pleases.
@param infos: Infos that were gathered at program call.
@return : The dictionary containing the names and their associated machines.
"""
sourlients = {}
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith(machine)
if machines != []:
#print machines
machine = str(machines).replace('[', '').replace(
']',
'').replace(',',
'').replace("'",
'').replace('"',
'').replace(" ", '')
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end,
machines,
pattern=None,
havingrunOnAllMachines=True)
else:
rxNames, txNames = [], []
else: #might be a groups machine,in this case do not force to have run on all machines.....
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end, [machine],
pattern=None,
havingrunOnAllMachines=False)
else:
rxNames, txNames = [], []
else: #not a cluster, une mahcine name directly. Force to have run exclusivly on specified machine.
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end, [machine],
pattern=None,
havingrunOnAllMachines=True)
else:
rxNames, txNames = [], []
if infos.fileType == "rx":
namesToAdd = rxNames
elif infos.fileType == "tx":
namesToAdd = txNames
for nameToAdd in namesToAdd:
if nameToAdd in sourlients.keys():
if machine not in sourlients[nameToAdd]:
sourlients[nameToAdd].append(machine)
else:
sourlients[nameToAdd] = [machine]
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
for group in groups:
if configParameters.groupParameters.groupFileTypes[
group] == infos.fileType:
sourlients[group] = [str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')]
return sourlients
