def getDataFromDatabases( sourlients, dataTypes, infos ): """ @summary: Gathers up all the requried data from allthe concerned databases @param sourlients: List of sources clients for wich we need to gather up data. @param machines: Machines on which the clients reside. @param dataTypes: Datatypes for which we need to collect data. @return : Return the data dictionary filled with all the collected data. """ data = {} for sourlient in sourlients.keys() : data[sourlient] = {} sourlientsMachines = sourlients[sourlient] for machine in infos.machinesToSearch : if infos.machinesAreClusters == True: machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machines = machineConfig.getMachinesAssociatedWith( machine ) oldMachine = machine machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\ .replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'') if machine == '': #print "trouvaille !!!" machine = oldMachine if machine in sourlientsMachines: data[sourlient][machine] = {} for dataType in dataTypes : if infos.outputLanguage != 'en' : translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage ) else : translatedDataType = dataType databaseName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients= [sourlient], machines = [machine], fileType = infos.fileType ) if not os.path.isfile( databaseName ): if infos.includegroups == True: databaseName = RrdUtilities.buildRRDFileName(dataType = translatedDataType, groupName = sourlient, machines = [machine], fileType = infos.fileType, usage = "group" ) lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType ) fetchedInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "fetchData" ) desiredInterval = getInterval( int(StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal = "plotGraphic" ) interval = desiredInterval minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\ int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\ fetchedInterval,desiredInterval, type = "average" ) data[sourlient][machine][dataType] = {} data[sourlient][machine][dataType]["min"] = minimum data[sourlient][machine][dataType]["max"] = maximum data[sourlient][machine][dataType]["mean"] = mean data[sourlient][machine][dataType]["total"] = total return data
def updateGroupedRoundRobinDatabases( infos, logger = None ): """ @summary : This method is to be used to update the database used to stored the merged data of a group. """ endTime = StatsDateLib.getSecondsSinceEpoch( infos.endTime ) tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients = infos.group, machines = infos.machines, fileType = infos.fileTypes[0] ) startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0] ) if startTime == 0 : startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( infos.endTime ) ) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), infos.clients, infos.fileTypes[0], infos.machines ) #print timeSeperators for i in xrange(0, len( timeSeperators ),2 ):#timeseperators should always be coming in pairs startTime = StatsDateLib.getSecondsSinceEpoch( timeSeperators[i] ) dataPairs = getPairs( infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i+1], infos.group, logger ) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = infos.group, groupName = infos.group, machines = infos.machines,fileType = infos.fileTypes[0], usage = "group" ) if not os.path.isfile( rrdFileName ): createRoundRobinDatabase( rrdFileName, startTime, dataType ) if endTime > startTime : j = 0 while dataPairs[ dataType ][j][0] < startTime and j < len( dataPairs[ dataType ] ): #print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime) j = j +1 for k in range ( j, len( dataPairs[ dataType ] ) ): #print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] )) try: rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) ) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) ) except: pass pass else: #print "endTime %s was not bigger than start time %s" %( endTime, startTime ) if logger != None : try: logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName ) except: pass RrdUtilities.setDatabaseTimeOfUpdate( tempRRDFileName, infos.fileTypes[0], endTime )
def updateRoundRobinDatabases( client, machines, fileType, endTime, logger = None ): """ @summary : This method updates every database linked to a certain client. @note : Database types are linked to the filetype associated with the client. """ combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in machines ] ) tempRRDFileName = RrdUtilities.buildRRDFileName( dataType = _("errors"), clients = [client], machines = machines, fileType = fileType) startTime = RrdUtilities.getDatabaseTimeOfUpdate( tempRRDFileName, fileType ) if startTime == 0 : startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight( endTime ) ) endTime = StatsDateLib.getSecondsSinceEpoch( endTime ) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory(StatsDateLib.getIsoFromEpoch( startTime ), StatsDateLib.getIsoFromEpoch( endTime ), [client], fileType, machines ) for i in xrange( len(timeSeperators) -1 ) : dataPairs = getPairs( [client], machines, fileType, timeSeperators[i], timeSeperators[i+1] , groupName = "", logger = logger ) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm(dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType = translatedDataType, clients = [client], machines = machines, fileType = fileType ) if not os.path.isfile( rrdFileName ): createRoundRobinDatabase( databaseName = rrdFileName , startTime= startTime, dataType = dataType ) if endTime > startTime : j = 0 while dataPairs[ dataType ][j][0] < startTime: j = j +1 for k in range ( j, len( dataPairs[ dataType ] ) ): try: rrdtool.update( rrdFileName, '%s:%s' %( int( dataPairs[ dataType ][k][0] ), dataPairs[ dataType ][k][1] ) ) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " %( rrdFileName,int( dataPairs[ dataType ][k][0] ) ) ) except: pass pass if logger != None : try: logger.info( _( "Updated %s db for %s in db named : %s" ) %( dataType, client, rrdFileName ) ) except: pass else: if logger != None : try: logger.warning( _( "This database was not updated since it's last update was more recent than specified date : %s" ) %rrdFileName ) except: pass RrdUtilities.setDatabaseTimeOfUpdate( rrdFileName, fileType, endTime )
def getDataFromDatabases(sourlients, dataTypes, infos): """ @summary: Gathers up all the requried data from allthe concerned databases @param sourlients: List of sources clients for wich we need to gather up data. @param machines: Machines on which the clients reside. @param dataTypes: Datatypes for which we need to collect data. @return : Return the data dictionary filled with all the collected data. """ data = {} for sourlient in sourlients.keys(): data[sourlient] = {} sourlientsMachines = sourlients[sourlient] for machine in infos.machinesToSearch: if infos.machinesAreClusters == True: machineConfig = MachineConfigParameters() machineConfig.getParametersFromMachineConfigurationFile() machines = machineConfig.getMachinesAssociatedWith(machine) oldMachine = machine machine = str(machines).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')\ .replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'') if machine == '': #print "trouvaille !!!" machine = oldMachine if machine in sourlientsMachines: data[sourlient][machine] = {} for dataType in dataTypes: if infos.outputLanguage != 'en': translatedDataType = LanguageTools.translateDataType( dataType, "en", infos.outputLanguage) else: translatedDataType = dataType databaseName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=[sourlient], machines=[machine], fileType=infos.fileType) if not os.path.isfile(databaseName): if infos.includegroups == True: databaseName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, groupName=sourlient, machines=[machine], fileType=infos.fileType, usage="group") lastUpdate = RrdUtilities.getDatabaseTimeOfUpdate( databaseName, infos.fileType) fetchedInterval = getInterval(int( StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal="fetchData") desiredInterval = getInterval(int( StatsDateLib.getSecondsSinceEpoch(infos.start)), lastUpdate, dataType, goal="plotGraphic") interval = desiredInterval minimum, maximum, mean, total = getGraphicsMinMaxMeanTotal( databaseName, int(StatsDateLib.getSecondsSinceEpoch(infos.start)),\ int(StatsDateLib.getSecondsSinceEpoch(infos.end)), infos.span,\ fetchedInterval,desiredInterval, type = "average" ) data[sourlient][machine][dataType] = {} data[sourlient][machine][dataType]["min"] = minimum data[sourlient][machine][dataType]["max"] = maximum data[sourlient][machine][dataType]["mean"] = mean data[sourlient][machine][dataType]["total"] = total return data
def updateGroupedRoundRobinDatabases(infos, logger=None): """ @summary : This method is to be used to update the database used to stored the merged data of a group. """ endTime = StatsDateLib.getSecondsSinceEpoch(infos.endTime) tempRRDFileName = RrdUtilities.buildRRDFileName( _("errors"), clients=infos.group, machines=infos.machines, fileType=infos.fileTypes[0]) startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0]) if startTime == 0: startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight(infos.endTime)) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch(startTime), StatsDateLib.getIsoFromEpoch(endTime), infos.clients, infos.fileTypes[0], infos.machines) #print timeSeperators for i in xrange(0, len(timeSeperators), 2): #timeseperators should always be coming in pairs startTime = StatsDateLib.getSecondsSinceEpoch(timeSeperators[i]) dataPairs = getPairs(infos.clients, infos.machines, infos.fileTypes[0], timeSeperators[i], timeSeperators[i + 1], infos.group, logger) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm( dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=infos.group, groupName=infos.group, machines=infos.machines, fileType=infos.fileTypes[0], usage="group") if not os.path.isfile(rrdFileName): createRoundRobinDatabase(rrdFileName, startTime, dataType) if endTime > startTime: j = 0 while dataPairs[dataType][j][0] < startTime and j < len( dataPairs[dataType]): #print "going over : %s startime was :%s" %(dataPairs[ dataType ][j][0], startTime) j = j + 1 for k in range(j, len(dataPairs[dataType])): #print "updating %s at %s" %(rrdFileName, int( dataPairs[ dataType ][k][0] )) try: rrdtool.update( rrdFileName, '%s:%s' % (int(dataPairs[dataType][k][0]), dataPairs[dataType][k][1])) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " % (rrdFileName, int(dataPairs[dataType][k][0]))) except: pass pass else: #print "endTime %s was not bigger than start time %s" %( endTime, startTime ) if logger != None: try: logger.warning( _("This database was not updated since it's last update was more recent than specified date : %s" ) % rrdFileName) except: pass RrdUtilities.setDatabaseTimeOfUpdate(tempRRDFileName, infos.fileTypes[0], endTime)
def updateRoundRobinDatabases(client, machines, fileType, endTime, logger=None): """ @summary : This method updates every database linked to a certain client. @note : Database types are linked to the filetype associated with the client. """ combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in machines]) tempRRDFileName = RrdUtilities.buildRRDFileName(dataType=_("errors"), clients=[client], machines=machines, fileType=fileType) startTime = RrdUtilities.getDatabaseTimeOfUpdate(tempRRDFileName, fileType) if startTime == 0: startTime = StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoTodaysMidnight(endTime)) endTime = StatsDateLib.getSecondsSinceEpoch(endTime) timeSeperators = getTimeSeperatorsBasedOnAvailableMemory( StatsDateLib.getIsoFromEpoch(startTime), StatsDateLib.getIsoFromEpoch(endTime), [client], fileType, machines) for i in xrange(len(timeSeperators) - 1): dataPairs = getPairs([client], machines, fileType, timeSeperators[i], timeSeperators[i + 1], groupName="", logger=logger) for dataType in dataPairs: translatedDataType = LanguageTools.translateTerm( dataType, 'en', LanguageTools.getMainApplicationLanguage(), CURRENT_MODULE_ABS_PATH) rrdFileName = RrdUtilities.buildRRDFileName( dataType=translatedDataType, clients=[client], machines=machines, fileType=fileType) if not os.path.isfile(rrdFileName): createRoundRobinDatabase(databaseName=rrdFileName, startTime=startTime, dataType=dataType) if endTime > startTime: j = 0 while dataPairs[dataType][j][0] < startTime: j = j + 1 for k in range(j, len(dataPairs[dataType])): try: rrdtool.update( rrdFileName, '%s:%s' % (int(dataPairs[dataType][k][0]), dataPairs[dataType][k][1])) except: if logger != None: try: logger.warning( "Could not update %s. Last update was more recent than %s " % (rrdFileName, int(dataPairs[dataType][k][0]))) except: pass pass if logger != None: try: logger.info( _("Updated %s db for %s in db named : %s") % (dataType, client, rrdFileName)) except: pass else: if logger != None: try: logger.warning( _("This database was not updated since it's last update was more recent than specified date : %s" ) % rrdFileName) except: pass RrdUtilities.setDatabaseTimeOfUpdate(rrdFileName, fileType, endTime)