def getRxTxNamesHavingRunDuringPeriod(start, end, machines, pattern=None, havingrunOnAllMachines=False): """ Browses all the rrd database directories to find the time of the last update of each databases. If database was updated between start and end and the client or source is from the specified machine, the name of the client or source is added to rxNames or txNames. """ rxNames = [] txNames = [] txOnlyDatabases = [] rxTxDatabases = [] combinedMachineName = "" start = StatsDateLib.getSecondsSinceEpoch(start) end = StatsDateLib.getSecondsSinceEpoch(end) if havingrunOnAllMachines == False: for machine in machines: rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_*%s*") % (STATSPATHS.STATSCURRENTDB, machine)) txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_*%s*") % (STATSPATHS.STATSCURRENTDB, machine)) #Keep only client/source names. for rxtxLongName in rxTxDatabasesLongNames: if pattern == None: if rxtxLongName not in rxTxDatabases: rxTxDatabases.append(rxtxLongName) else: if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern): if rxtxLongName not in rxTxDatabases: rxTxDatabases.append(rxtxLongName) for txLongName in txOnlyDatabasesLongNames: if pattern == None: if txLongName not in txOnlyDatabases: txOnlyDatabases.append(txLongName) else: if fnmatch.fnmatch(os.path.basename(txLongName), pattern): if txLongName not in txOnlyDatabases: txOnlyDatabases.append(txLongName) else: for machine in machines: combinedMachineName = combinedMachineName + machine rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_%s*") % (STATSPATHS.STATSCURRENTDB, combinedMachineName)) txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_%s*") % (STATSPATHS.STATSCURRENTDB, combinedMachineName)) #Keep only client/source names. for rxtxLongName in rxTxDatabasesLongNames: if pattern == None: rxTxDatabases.append(rxtxLongName) else: if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern): rxTxDatabases.append(rxtxLongName) for txLongName in txOnlyDatabasesLongNames: if pattern == None: txOnlyDatabases.append(txLongName) else: if fnmatch.fnmatch(os.path.basename(txLongName), pattern): txOnlyDatabases.append(txLongName) rxOnlyDatabases = filter(lambda x: x not in txOnlyDatabases, rxTxDatabases) for rxDatabase in rxOnlyDatabases: lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(rxDatabase, "rx") if lastUpdate >= start: #fileName format is ../path/rxName_machineName rxDatabase = os.path.basename(rxDatabase) rxDatabase = rxDatabase.split( "_%s" % (rxDatabase.split('_')[-1:][0]))[0] rxNames.append(rxDatabase) for txDatabase in txOnlyDatabases: lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate(txDatabase, "tx") if lastUpdate >= start: txDatabase = os.path.basename(txDatabase) txDatabase = txDatabase.split( "_%s" % (txDatabase.split('_')[-1:][0]))[0] txNames.append(txDatabase) rxNames = filter(GeneralStatsLibraryMethods.filterGroupNames, rxNames) txNames = filter(GeneralStatsLibraryMethods.filterGroupNames, txNames) try: rxNames.remove('rx') except: pass try: txNames.remove('tx') except: pass rxNames.sort() txNames.sort() return rxNames, txNames
def getRxTxNamesHavingRunDuringPeriod( start, end, machines, pattern = None, havingrunOnAllMachines = False ): """ Browses all the rrd database directories to find the time of the last update of each databases. If database was updated between start and end and the client or source is from the specified machine, the name of the client or source is added to rxNames or txNames. """ rxNames = [] txNames = [] txOnlyDatabases = [] rxTxDatabases = [] combinedMachineName = "" start = StatsDateLib.getSecondsSinceEpoch(start) end = StatsDateLib.getSecondsSinceEpoch(end) if havingrunOnAllMachines == False: for machine in machines: rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_*%s*") %( STATSPATHS.STATSCURRENTDB, machine ) ) txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_*%s*") %( STATSPATHS.STATSCURRENTDB, machine ) ) #Keep only client/source names. for rxtxLongName in rxTxDatabasesLongNames: if pattern == None: if rxtxLongName not in rxTxDatabases: rxTxDatabases.append( rxtxLongName ) else: if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern ): if rxtxLongName not in rxTxDatabases: rxTxDatabases.append( rxtxLongName ) for txLongName in txOnlyDatabasesLongNames: if pattern == None: if txLongName not in txOnlyDatabases: txOnlyDatabases.append( txLongName ) else: if fnmatch.fnmatch(os.path.basename(txLongName), pattern): if txLongName not in txOnlyDatabases: txOnlyDatabases.append( txLongName ) else: for machine in machines: combinedMachineName = combinedMachineName + machine rxTxDatabasesLongNames = glob.glob( _("%sbytecount/*_%s*") %( STATSPATHS.STATSCURRENTDB, combinedMachineName ) ) txOnlyDatabasesLongNames = glob.glob( _("%slatency/*_%s*") %( STATSPATHS.STATSCURRENTDB, combinedMachineName ) ) #Keep only client/source names. for rxtxLongName in rxTxDatabasesLongNames: if pattern == None: rxTxDatabases.append( rxtxLongName ) else: if fnmatch.fnmatch(os.path.basename(rxtxLongName), pattern ): rxTxDatabases.append( rxtxLongName ) for txLongName in txOnlyDatabasesLongNames: if pattern == None: txOnlyDatabases.append( txLongName ) else: if fnmatch.fnmatch(os.path.basename(txLongName), pattern): txOnlyDatabases.append( txLongName ) rxOnlyDatabases = filter( lambda x: x not in txOnlyDatabases, rxTxDatabases ) for rxDatabase in rxOnlyDatabases: lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( rxDatabase, "rx" ) if lastUpdate >= start: #fileName format is ../path/rxName_machineName rxDatabase = os.path.basename( rxDatabase ) rxDatabase = rxDatabase.split( "_%s" %( rxDatabase.split('_')[-1:][0] ) )[0] rxNames.append( rxDatabase ) for txDatabase in txOnlyDatabases: lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( txDatabase, "tx" ) if lastUpdate >= start: txDatabase = os.path.basename( txDatabase ) txDatabase = txDatabase.split("_%s" %( txDatabase.split('_')[-1:][0] ) )[0] txNames.append( txDatabase ) rxNames = filter( GeneralStatsLibraryMethods.filterGroupNames, rxNames ) txNames = filter( GeneralStatsLibraryMethods.filterGroupNames, txNames ) try: rxNames.remove('rx') except: pass try: txNames.remove('tx') except: pass rxNames.sort() txNames.sort() return rxNames, txNames
def getDataFromDatabases( sourlients, dataTypes, infos ): """ @summary: Gathers up all the requried data from allthe concerned databases @param sourlients: List of sources clients for wich we need to gather up data. @param machines: Machines on which the clients reside. @param dataTypes: Datatypes for which we need to collect data. @return : Return the data dictionary filled with all the collected data. """ data = {} for sourlient in sourlients.keys() : data[sourlient] = {} sourlientsMachines = sourlients[sourlient] for machine in infos.machinesToSearch : if infos.machinesAreClusters == True: machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machines = machineConfig.getMachinesAssociatedWith( machine ) oldMachine = machine machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\ .replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'') if machine == '': #print "trouvaille !!!" machine = oldMachine if machine in sourlientsMachines: data[sourlient][machine] = {} for dataType in dataTypes : if infos.outputLanguage != 'en' : translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage ) else : translatedDataType = dataType databaseName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients= [sourlient], machines = [machine], fileType = infos.fileType ) if not os.path.isfile( databaseName ): if infos.includegroups == True: databaseName = RrdUtilities.buildRRDFileName(dataType = translatedDataType, groupName = sourlient, machines = [machine], fileType = infos.fileType, usage = "group" ) lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType ) fetchedInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "fetchData" ) desiredInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "plotGraphic" ) interval = desiredInterval minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\ int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\ fetchedInterval,desiredInterval, type = "average" ) data[sourlient][machine][dataType] = {} data[sourlient][machine][dataType]["min"] = minimum data[sourlient][machine][dataType]["max"] = maximum data[sourlient][machine][dataType]["mean"] = mean data[sourlient][machine][dataType]["total"] = total return data
def updateGroupedRoundRobinDatabases( infos, logger = None ): """ @summary : This method is to be used to update the database used to stored the merged data of a group. """ endTime = StatsDateLib.getSecondsSinceEpoch( infos.endTime ) tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients = infos.group, machines = infos.machines, fileType = infos.fileTypes[0] ) startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0] ) if startTime == 0 : startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( infos.endTime ) ) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), infos.clients, infos.fileTypes[0], infos.machines ) #print timeSeperators for i in xrange(0, len( timeSeperators ),2 ):#timeseperators should always be coming in pairs startTime = StatsDateLib.getSecondsSinceEpoch( timeSeperators[i] ) dataPairs = getPairs( infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i+1], infos.group, logger ) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = infos.group, groupName = infos.group, machines = infos.machines,fileType = infos.fileTypes[0], usage = "group" ) if not os.path.isfile( rrdFileName ): createRoundRobinDatabase( rrdFileName, startTime, dataType ) if endTime > startTime : j = 0 while dataPairs[ dataType ][j][0] < startTime and j < len( dataPairs[ dataType ] ): #print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime) j = j +1 for k in range ( j, len( dataPairs[ dataType ] ) ): #print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] )) try: rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) ) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) ) except: pass pass else: #print "endTime %s was not bigger than start time %s" %( endTime, startTime ) if logger != None : try: logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName ) except: pass RrdUtilities.setDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0], endTime )
def updateRoundRobinDatabases( client, machines, fileType, endTime, logger = None ): """ @summary : This method updates every database linked to a certain client. @note : Database types are linked to the filetype associated with the client. """ combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in machines ] ) tempRRDFileName = RrdUtilities.buildRRDFileName( dataType = _("errors"), clients = [client], machines = machines, fileType = fileType) startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, fileType ) if startTime == 0 : startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( endTime ) ) endTime = StatsDateLib.getSecondsSinceEpoch( endTime ) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), [client], fileType, machines ) for i in xrange( len(timeSeperators) -1 ) : dataPairs = getPairs( [client], machines, fileType, timeSeperators[i], timeSeperators[i+1] , groupName = "", logger = logger ) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = [client], machines = machines, fileType = fileType ) if not os.path.isfile( rrdFileName ): createRoundRobinDatabase( databaseName = rrdFileName , startTime= startTime, dataType = dataType ) if endTime > startTime : j = 0 while dataPairs[ dataType ][j][0] < startTime: j = j +1 for k in range ( j, len( dataPairs[ dataType ] ) ): try: rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) ) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) ) except: pass pass if logger != None : try: logger.info( _( "Updated %s db for %s in db named : %s" ) %( dataType, client, rrdFileName ) ) except: pass else: if logger != None : try: logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName ) except: pass RrdUtilities.setDatabaseTimeOfUpdate( rrdFileName, fileType, endTime )
def getDataFromDatabases(sourlients, dataTypes, infos): """ @summary: Gathers up all the requried data from allthe concerned databases @param sourlients: List of sources clients for wich we need to gather up data. @param machines: Machines on which the clients reside. @param dataTypes: Datatypes for which we need to collect data. @return : Return the data dictionary filled with all the collected data. """ data = {} for sourlient in sourlients.keys(): data[sourlient] = {} sourlientsMachines = sourlients[sourlient] for machine in infos.machinesToSearch: if infos.machinesAreClusters == True: machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machines = machineConfig.getMachinesAssociatedWith(machine) oldMachine = machine machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\ .replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'') if machine == '': #print "trouvaille !!!" machine = oldMachine if machine in sourlientsMachines: data[sourlient][machine] = {} for dataType in dataTypes: if infos.outputLanguage != 'en': translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage) else: translatedDataType = dataType databaseName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=[sourlient], machines=[machine], fileType=infos.fileType) if not os.path.isfile(databaseName): if infos.includegroups == True: databaseName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, groupName=sourlient, machines=[machine], fileType=infos.fileType, usage="group") lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType) fetchedInterval = getInterval(int( StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal="fetchData") desiredInterval = getInterval(int( StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal="plotGraphic") interval = desiredInterval minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\ int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\ fetchedInterval,desiredInterval, type = "average" ) data[sourlient][machine][dataType] = {} data[sourlient][machine][dataType]["min"] = minimum data[sourlient][machine][dataType]["max"] = maximum data[sourlient][machine][dataType]["mean"] = mean data[sourlient][machine][dataType]["total"] = total return data
def updateGroupedRoundRobinDatabases(infos, logger=None): """ @summary : This method is to be used to update the database used to stored the merged data of a group. """ endTime = StatsDateLib.getSecondsSinceEpoch(infos.endTime) tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients=infos.group, machines=infos.machines, fileType=infos.fileTypes[0]) startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0]) if startTime == 0: startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight(infos.endTime)) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch(startTime), StatsDateLib.getIsoFromEpoch(endTime), infos.clients, infos.fileTypes[0], infos.machines) #print timeSeperators for i in xrange(0, len(timeSeperators), 2): #timeseperators should always be coming in pairs startTime = StatsDateLib.getSecondsSinceEpoch(timeSeperators[i]) dataPairs = getPairs(infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i + 1], infos.group, logger) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm( dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=infos.group, groupName=infos.group, machines=infos.machines, fileType=infos.fileTypes[0], usage="group") if not os.path.isfile(rrdFileName): createRoundRobinDatabase(rrdFileName, startTime, dataType) if endTime > startTime: j = 0 while dataPairs[dataType][j][0] < startTime and j < len( dataPairs[dataType]): #print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime) j = j + 1 for k in range(j, len(dataPairs[dataType])): #print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] )) try: rrdtool.update( rrdFileName, '%s:%s' % (int(dataPairs[dataType][k][0]), dataPairs[dataType][k][1])) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " % (rrdFileName, int(dataPairs[dataType][k][0]))) except: pass pass else: #print "endTime %s was not bigger than start time %s" %( endTime, startTime ) if logger != None: try: logger.warning( _("This database was not updated since it's last update was more recent than specified date : %s" ) % rrdFileName) except: pass RrdUtilities.setDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0], endTime)
def updateRoundRobinDatabases(client, machines, fileType, endTime, logger=None): """ @summary : This method updates every database linked to a certain client. @note : Database types are linked to the filetype associated with the client. """ combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in machines]) tempRRDFileName = RrdUtilities.buildRRDFileName(dataType=_("errors"), clients=[client], machines=machines, fileType=fileType) startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, fileType) if startTime == 0: startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight(endTime)) endTime = StatsDateLib.getSecondsSinceEpoch(endTime) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch(startTime), StatsDateLib.getIsoFromEpoch(endTime), [client], fileType, machines) for i in xrange(len(timeSeperators) - 1): dataPairs = getPairs([client], machines, fileType, timeSeperators[i], timeSeperators[i + 1], groupName="", logger=logger) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm( dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=[client], machines=machines, fileType=fileType) if not os.path.isfile(rrdFileName): createRoundRobinDatabase(databaseName=rrdFileName, startTime=startTime, dataType=dataType) if endTime > startTime: j = 0 while dataPairs[dataType][j][0] < startTime: j = j + 1 for k in range(j, len(dataPairs[dataType])): try: rrdtool.update( rrdFileName, '%s:%s' % (int(dataPairs[dataType][k][0]), dataPairs[dataType][k][1])) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " % (rrdFileName, int(dataPairs[dataType][k][0]))) except: pass pass if logger != None: try: logger.info( _("Updated %s db for %s in db named : %s") % (dataType, client, rrdFileName)) except: pass else: if logger != None: try: logger.warning( _("This database was not updated since it's last update was more recent than specified date : %s" ) % rrdFileName) except: pass RrdUtilities.setDatabaseTimeOfUpdate(rrdFileName, fileType, endTime)