def getRxTxNamesHavingRunDuringPeriod(start,
end,
machines,
pattern=None,
havingrunOnAllMachines=False):
"""
Browses all the rrd database directories to find
the time of the last update of each databases.
If database was updated between start and end
and the client or source is from the specified
machine, the name of the client or source is
added to rxNames or txNames.
"""
rxNames = []
txNames = []
txOnlyDatabases = []
rxTxDatabases = []
combinedMachineName = ""
start = StatsDateLib.getSecondsSinceEpoch(start)
end = StatsDateLib.getSecondsSinceEpoch(end)
if havingrunOnAllMachines == False:
for machine in machines:
rxTxDatabasesLongNames = glob.glob(
_("%sbytecount/*_*%s*") %
(STATSPATHS.STATSCURRENTDB, machine))
txOnlyDatabasesLongNames = glob.glob(
_("%slatency/*_*%s*") %
(STATSPATHS.STATSCURRENTDB, machine))
#Keep only client/source names.
for rxtxLongName in rxTxDatabasesLongNames:
if pattern == None:
if rxtxLongName not in rxTxDatabases:
rxTxDatabases.append(rxtxLongName)
else:
if fnmatch.fnmatch(os.path.basename(rxtxLongName),
pattern):
if rxtxLongName not in rxTxDatabases:
rxTxDatabases.append(rxtxLongName)
for txLongName in txOnlyDatabasesLongNames:
if pattern == None:
if txLongName not in txOnlyDatabases:
txOnlyDatabases.append(txLongName)
else:
if fnmatch.fnmatch(os.path.basename(txLongName),
pattern):
if txLongName not in txOnlyDatabases:
txOnlyDatabases.append(txLongName)
else:
for machine in machines:
combinedMachineName = combinedMachineName + machine
rxTxDatabasesLongNames = glob.glob(
_("%sbytecount/*_%s*") %
(STATSPATHS.STATSCURRENTDB, combinedMachineName))
txOnlyDatabasesLongNames = glob.glob(
_("%slatency/*_%s*") %
(STATSPATHS.STATSCURRENTDB, combinedMachineName))
#Keep only client/source names.
for rxtxLongName in rxTxDatabasesLongNames:
if pattern == None:
rxTxDatabases.append(rxtxLongName)
else:
if fnmatch.fnmatch(os.path.basename(rxtxLongName),
pattern):
rxTxDatabases.append(rxtxLongName)
for txLongName in txOnlyDatabasesLongNames:
if pattern == None:
txOnlyDatabases.append(txLongName)
else:
if fnmatch.fnmatch(os.path.basename(txLongName), pattern):
txOnlyDatabases.append(txLongName)
rxOnlyDatabases = filter(lambda x: x not in txOnlyDatabases,
rxTxDatabases)
for rxDatabase in rxOnlyDatabases:
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(rxDatabase, "rx")
if lastUpdate >= start:
#fileName format is ../path/rxName_machineName
rxDatabase = os.path.basename(rxDatabase)
rxDatabase = rxDatabase.split(
"_%s" % (rxDatabase.split('_')[-1:][0]))[0]
rxNames.append(rxDatabase)
for txDatabase in txOnlyDatabases:
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(txDatabase, "tx")
if lastUpdate >= start:
txDatabase = os.path.basename(txDatabase)
txDatabase = txDatabase.split(
"_%s" % (txDatabase.split('_')[-1:][0]))[0]
txNames.append(txDatabase)
rxNames = filter(GeneralStatsLibraryMethods.filterGroupNames, rxNames)
txNames = filter(GeneralStatsLibraryMethods.filterGroupNames, txNames)
try:
rxNames.remove('rx')
except:
pass
try:
txNames.remove('tx')
except:
pass
rxNames.sort()
txNames.sort()
return rxNames, txNames
def getRxTxNamesHavingRunDuringPeriod( start, end, machines, pattern = None, havingrunOnAllMachines = False ):
"""
Browses all the rrd database directories to find
the time of the last update of each databases.
If database was updated between start and end
and the client or source is from the specified
machine, the name of the client or source is
added to rxNames or txNames.
"""
rxNames = []
txNames = []
txOnlyDatabases = []
rxTxDatabases = []
combinedMachineName = ""
start = StatsDateLib.getSecondsSinceEpoch(start)
end = StatsDateLib.getSecondsSinceEpoch(end)
if havingrunOnAllMachines == False:
for machine in machines:
rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_*%s*") %( STATSPATHS.STATSCURRENTDB, machine ) )
txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_*%s*") %( STATSPATHS.STATSCURRENTDB, machine ) )
#Keep only client/source names.
for rxtxLongName in rxTxDatabasesLongNames:
if pattern == None:
if rxtxLongName not in rxTxDatabases:
rxTxDatabases.append( rxtxLongName )
else:
if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern ):
if rxtxLongName not in rxTxDatabases:
rxTxDatabases.append( rxtxLongName )
for txLongName in txOnlyDatabasesLongNames:
if pattern == None:
if txLongName not in txOnlyDatabases:
txOnlyDatabases.append( txLongName )
else:
if fnmatch.fnmatch(os.path.basename(txLongName), pattern):
if txLongName not in txOnlyDatabases:
txOnlyDatabases.append( txLongName )
else:
for machine in machines:
combinedMachineName = combinedMachineName + machine
rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_%s*") %( STATSPATHS.STATSCURRENTDB, combinedMachineName ) )
txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_%s*") %( STATSPATHS.STATSCURRENTDB, combinedMachineName ) )
#Keep only client/source names.
for rxtxLongName in rxTxDatabasesLongNames:
if pattern == None:
rxTxDatabases.append( rxtxLongName )
else:
if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern ):
rxTxDatabases.append( rxtxLongName )
for txLongName in txOnlyDatabasesLongNames:
if pattern == None:
txOnlyDatabases.append( txLongName )
else:
if fnmatch.fnmatch(os.path.basename(txLongName), pattern):
txOnlyDatabases.append( txLongName )
rxOnlyDatabases = filter( lambda x: x not in txOnlyDatabases, rxTxDatabases )
for rxDatabase in rxOnlyDatabases:
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( rxDatabase, "rx" )
if lastUpdate >= start:
#fileName format is ../path/rxName_machineName
rxDatabase = os.path.basename( rxDatabase )
rxDatabase = rxDatabase.split( "_%s" %( rxDatabase.split('_')[-1:][0] ) )[0]
rxNames.append( rxDatabase )
for txDatabase in txOnlyDatabases:
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( txDatabase, "tx" )
if lastUpdate >= start:
txDatabase = os.path.basename( txDatabase )
txDatabase = txDatabase.split("_%s" %( txDatabase.split('_')[-1:][0] ) )[0]
txNames.append( txDatabase )
rxNames = filter( GeneralStatsLibraryMethods.filterGroupNames, rxNames )
txNames = filter( GeneralStatsLibraryMethods.filterGroupNames, txNames )
try:
rxNames.remove('rx')
except:
pass
try:
txNames.remove('tx')
except:
pass
rxNames.sort()
txNames.sort()
return rxNames, txNames
def getDataFromDatabases( sourlients, dataTypes, infos ):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys() :
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch :
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith( machine )
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes :
if infos.outputLanguage != 'en' :
translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage )
else :
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients= [sourlient], machines = [machine], fileType = infos.fileType )
if not os.path.isfile( databaseName ):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(dataType = translatedDataType, groupName = sourlient, machines = [machine], fileType = infos.fileType, usage = "group" )
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType )
fetchedInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "fetchData" )
desiredInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "plotGraphic" )
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def updateGroupedRoundRobinDatabases( infos, logger = None ):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch( infos.endTime )
tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients = infos.group, machines = infos.machines, fileType = infos.fileTypes[0] )
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0] )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( infos.endTime ) )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), infos.clients, infos.fileTypes[0], infos.machines )
#print timeSeperators
for i in xrange(0, len( timeSeperators ),2 ):#timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch( timeSeperators[i] )
dataPairs = getPairs( infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i+1], infos.group, logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = infos.group, groupName = infos.group, machines = infos.machines,fileType = infos.fileTypes[0], usage = "group" )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( rrdFileName, startTime, dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime and j < len( dataPairs[ dataType ] ):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0], endTime )
def updateRoundRobinDatabases( client, machines, fileType, endTime, logger = None ):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join( [machine for machine in machines ] )
tempRRDFileName = RrdUtilities.buildRRDFileName( dataType = _("errors"), clients = [client], machines = machines, fileType = fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, fileType )
if startTime == 0 :
startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( endTime ) )
endTime = StatsDateLib.getSecondsSinceEpoch( endTime )
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), [client], fileType, machines )
for i in xrange( len(timeSeperators) -1 ) :
dataPairs = getPairs( [client], machines, fileType, timeSeperators[i], timeSeperators[i+1] , groupName = "", logger = logger )
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = [client], machines = machines, fileType = fileType )
if not os.path.isfile( rrdFileName ):
createRoundRobinDatabase( databaseName = rrdFileName , startTime= startTime, dataType = dataType )
if endTime > startTime :
j = 0
while dataPairs[ dataType ][j][0] < startTime:
j = j +1
for k in range ( j, len( dataPairs[ dataType ] ) ):
try:
rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) )
except:
if logger != None:
try:
logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) )
except:
pass
pass
if logger != None :
try:
logger.info( _( "Updated %s db for %s in db named : %s" ) %( dataType, client, rrdFileName ) )
except:
pass
else:
if logger != None :
try:
logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName )
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate( rrdFileName, fileType, endTime )
def getDataFromDatabases(sourlients, dataTypes, infos):
"""
@summary: Gathers up all the requried data from allthe concerned databases
@param sourlients: List of sources clients for wich we need to gather up data.
@param machines: Machines on which the clients reside.
@param dataTypes: Datatypes for which we need to collect data.
@return : Return the data dictionary filled with all the collected data.
"""
data = {}
for sourlient in sourlients.keys():
data[sourlient] = {}
sourlientsMachines = sourlients[sourlient]
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith(machine)
oldMachine = machine
machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\
.replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine == '':
#print "trouvaille !!!"
machine = oldMachine
if machine in sourlientsMachines:
data[sourlient][machine] = {}
for dataType in dataTypes:
if infos.outputLanguage != 'en':
translatedDataType = LanguageTools.translateDataType(
dataType, "en", infos.outputLanguage)
else:
translatedDataType = dataType
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[sourlient],
machines=[machine],
fileType=infos.fileType)
if not os.path.isfile(databaseName):
if infos.includegroups == True:
databaseName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
groupName=sourlient,
machines=[machine],
fileType=infos.fileType,
usage="group")
lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(
databaseName, infos.fileType)
fetchedInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="fetchData")
desiredInterval = getInterval(int(
StatsDateLib.getSecondsSinceEpoch(infos.start)),
lastUpdate,
dataType,
goal="plotGraphic")
interval = desiredInterval
minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\
int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\
fetchedInterval,desiredInterval, type = "average" )
data[sourlient][machine][dataType] = {}
data[sourlient][machine][dataType]["min"] = minimum
data[sourlient][machine][dataType]["max"] = maximum
data[sourlient][machine][dataType]["mean"] = mean
data[sourlient][machine][dataType]["total"] = total
return data
def updateGroupedRoundRobinDatabases(infos, logger=None):
"""
@summary : This method is to be used to update the database
used to stored the merged data of a group.
"""
endTime = StatsDateLib.getSecondsSinceEpoch(infos.endTime)
tempRRDFileName = RrdUtilities.buildRRDFileName(
_("errors"),
clients=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0])
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName,
infos.fileTypes[0])
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(infos.endTime))
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), infos.clients,
infos.fileTypes[0], infos.machines)
#print timeSeperators
for i in xrange(0, len(timeSeperators),
2): #timeseperators should always be coming in pairs
startTime = StatsDateLib.getSecondsSinceEpoch(timeSeperators[i])
dataPairs = getPairs(infos.clients, infos.machines, infos.fileTypes[0],
timeSeperators[i], timeSeperators[i + 1],
infos.group, logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=infos.group,
groupName=infos.group,
machines=infos.machines,
fileType=infos.fileTypes[0],
usage="group")
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(rrdFileName, startTime, dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime and j < len(
dataPairs[dataType]):
#print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime)
j = j + 1
for k in range(j, len(dataPairs[dataType])):
#print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] ))
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
else:
#print "endTime %s was not bigger than start time %s" %( endTime, startTime )
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0],
endTime)
def updateRoundRobinDatabases(client,
machines,
fileType,
endTime,
logger=None):
"""
@summary : This method updates every database linked to a certain client.
@note : Database types are linked to the filetype associated with the client.
"""
combinedMachineName = ""
combinedMachineName = combinedMachineName.join(
[machine for machine in machines])
tempRRDFileName = RrdUtilities.buildRRDFileName(dataType=_("errors"),
clients=[client],
machines=machines,
fileType=fileType)
startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, fileType)
if startTime == 0:
startTime = StatsDateLib.getSecondsSinceEpoch(
StatsDateLib.getIsoTodaysMidnight(endTime))
endTime = StatsDateLib.getSecondsSinceEpoch(endTime)
timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(
StatsDateLib.getIsoFromEpoch(startTime),
StatsDateLib.getIsoFromEpoch(endTime), [client], fileType, machines)
for i in xrange(len(timeSeperators) - 1):
dataPairs = getPairs([client],
machines,
fileType,
timeSeperators[i],
timeSeperators[i + 1],
groupName="",
logger=logger)
for dataType in dataPairs:
translatedDataType = LanguageTools.translateTerm(
dataType, 'en', LanguageTools.getMainApplicationLanguage(),
CURRENT_MODULE_ABS_PATH)
rrdFileName = RrdUtilities.buildRRDFileName(
dataType=translatedDataType,
clients=[client],
machines=machines,
fileType=fileType)
if not os.path.isfile(rrdFileName):
createRoundRobinDatabase(databaseName=rrdFileName,
startTime=startTime,
dataType=dataType)
if endTime > startTime:
j = 0
while dataPairs[dataType][j][0] < startTime:
j = j + 1
for k in range(j, len(dataPairs[dataType])):
try:
rrdtool.update(
rrdFileName,
'%s:%s' % (int(dataPairs[dataType][k][0]),
dataPairs[dataType][k][1]))
except:
if logger != None:
try:
logger.warning(
"Could not update %s. Last update was more recent than %s "
% (rrdFileName,
int(dataPairs[dataType][k][0])))
except:
pass
pass
if logger != None:
try:
logger.info(
_("Updated %s db for %s in db named : %s") %
(dataType, client, rrdFileName))
except:
pass
else:
if logger != None:
try:
logger.warning(
_("This database was not updated since it's last update was more recent than specified date : %s"
) % rrdFileName)
except:
pass
RrdUtilities.setDatabaseTimeOfUpdate(rrdFileName, fileType,
endTime)
