def getDataFromDatabases( sourlients, dataTypes, infos ):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys() :
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch :
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith( machine )
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes :
if infos.outputLanguage != 'en' :
translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage )
else :
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients= [sourlient], machines = [machine], fileType = infos.fileType )
if not os.path.isfile( databaseName ):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(dataType = translatedDataType, groupName = sourlient, machines = [machine], fileType = infos.fileType, usage = "group" )
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType )
fetchedInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "fetchData" )
desiredInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "plotGraphic" )
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def updateGroupedRoundRobinDatabases( infos, logger = None ):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch( infos.endTime )
tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients = infos.group, machines = infos.machines, fileType = infos.fileTypes[0] )
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0] )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( infos.endTime ) )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), infos.clients, infos.fileTypes[0], infos.machines )
#print timeSeperators
for i in xrange(0, len( timeSeperators ),2 ):#timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch( timeSeperators[i] )
dataPairs = getPairs( infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i+1], infos.group, logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = infos.group, groupName = infos.group, machines = infos.machines,fileType = infos.fileTypes[0], usage = "group" )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( rrdFileName, startTime, dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime and j < len( dataPairs[ dataType ] ):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0], endTime )
def updateRoundRobinDatabases( client, machines, fileType, endTime, logger = None ):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join( [machine for machine in machines ] )
tempRRDFileName = RrdUtilities.buildRRDFileName( dataType = _("errors"), clients = [client], machines = machines, fileType = fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, fileType )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( endTime ) )
endTime = StatsDateLib.getSecondsSinceEpoch( endTime )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), [client], fileType, machines )
for i in xrange( len(timeSeperators) -1 ) :
dataPairs = getPairs( [client], machines, fileType, timeSeperators[i], timeSeperators[i+1] , groupName = "", logger = logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = [client], machines = machines, fileType = fileType )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( databaseName = rrdFileName , startTime= startTime, dataType = dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime:
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
if logger != None :
try:
logger.info( _( "Updated %s db for %s in db named : %s" ) %( dataType, client, rrdFileName ) )
except:
pass
else:
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( rrdFileName, fileType, endTime )
def getDataFromDatabases(sourlients, dataTypes, infos):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys():
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith(machine)
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes:
if infos.outputLanguage != 'en':
translatedDataType = LanguageTools.translateDataType(
dataType, "en", infos.outputLanguage)
else:
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[sourlient],
machines=[machine],
fileType=infos.fileType)
if not os.path.isfile(databaseName):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
groupName=sourlient,
machines=[machine],
fileType=infos.fileType,
usage="group")
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(
databaseName, infos.fileType)
fetchedInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="fetchData")
desiredInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="plotGraphic")
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def updateGroupedRoundRobinDatabases(infos, logger=None):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch(infos.endTime)
tempRRDFileName = RrdUtilities.buildRRDFileName(
_("errors"),
clients=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0])
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName,
infos.fileTypes[0])
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(infos.endTime))
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), infos.clients,
infos.fileTypes[0], infos.machines)
#print timeSeperators
for i in xrange(0, len(timeSeperators),
2): #timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch(timeSeperators[i])
dataPairs = getPairs(infos.clients, infos.machines, infos.fileTypes[0],
timeSeperators[i], timeSeperators[i + 1],
infos.group, logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=infos.group,
groupName=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0],
usage="group")
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(rrdFileName, startTime, dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime and j < len(
dataPairs[dataType]):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j + 1
for k in range(j, len(dataPairs[dataType])):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0],
endTime)
def updateRoundRobinDatabases(client,
machines,
fileType,
endTime,
logger=None):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join(
[machine for machine in machines])
tempRRDFileName = RrdUtilities.buildRRDFileName(dataType=_("errors"),
clients=[client],
machines=machines,
fileType=fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, fileType)
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(endTime))
endTime = StatsDateLib.getSecondsSinceEpoch(endTime)
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), [client], fileType, machines)
for i in xrange(len(timeSeperators) - 1):
dataPairs = getPairs([client],
machines,
fileType,
timeSeperators[i],
timeSeperators[i + 1],
groupName="",
logger=logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[client],
machines=machines,
fileType=fileType)
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(databaseName=rrdFileName,
startTime=startTime,
dataType=dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime:
j = j + 1
for k in range(j, len(dataPairs[dataType])):
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
if logger != None:
try:
logger.info(
_("Updated %s db for %s in db named : %s") %
(dataType, client, rrdFileName))
except:
pass
else:
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(rrdFileName, fileType,
endTime)
