def getMergedData( clients, fileType, machines, startTime, endTime, groupName = "", logger = None ): """ This method returns all data comprised between startTime and endTime as to be able to build pairs. """ if fileType == "tx": types = [ "errors","bytecount","latency", ] else: types = [ "errors","bytecount" ] #print startTime, endTime if len( machines ) > 1 or len( clients) > 1: statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = logger , startTime = startTime, endTime = endTime, clients = clients, fileType = fileType, machines = machines, groupName = groupName ) else:#only one machine, only merge different hours together statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = logger , startTime = startTime, endTime = endTime, client = clients[0], fileType = fileType, machine = machines[0] ) combinedMachineName = "" for machine in machines: combinedMachineName = combinedMachineName + machine dataCollector = StatsPickler( client = clients[0], statsTypes = types, directory = "", statsCollection = statsCollection, machine = combinedMachineName, logger = logger ) return dataCollector
def getMergedData(clients, fileType, machines, startTime, endTime, groupName="", logger=None): """ This method returns all data comprised between startTime and endTime as to be able to build pairs. """ if fileType == "tx": types = [ "errors", "bytecount", "latency", ] else: types = ["errors", "bytecount"] #print startTime, endTime if len(machines) > 1 or len(clients) > 1: statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger=logger, startTime=startTime, endTime=endTime, clients=clients, fileType=fileType, machines=machines, groupName=groupName) else: #only one machine, only merge different hours together statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger=logger, startTime=startTime, endTime=endTime, client=clients[0], fileType=fileType, machine=machines[0]) combinedMachineName = "" for machine in machines: combinedMachineName = combinedMachineName + machine dataCollector = StatsPickler(client=clients[0], statsTypes=types, directory="", statsCollection=statsCollection, machine=combinedMachineName, logger=logger) return dataCollector
def collectDataForIndividualGraphics(self, startTime, endTime, types): #find parameters """ @summary : Returns a list of ClientStatsPicklers instances, each of wich contains data for all the individual graphics. @return : List of ClientStatsPicklers instances. """ global _ dataCollection = [] for client in self.clientNames: # #Gather data from all previously created pickles.... if self.logger != None: try: self.logger.debug(_("Call to mergeHourlyPickles.")) self.logger.debug( _("Parameters used : %s %s %s") % (startTime, endTime, client)) except: pass if len(self.machines) > 1: clientArray = [] clientArray.append(client) statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\ clients = clientArray, fileType = self.fileType,\ machines = self.machines, groupName = self.groupName ) else: #only one machine, only merge different hours together statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = None , startTime = startTime, endTime = endTime,\ client = client, fileType = self.fileType, machine = self.machines[0] ) combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in self.machines]) dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\ statsCollection = statsCollection, machine = combinedMachineName,\ logger = None, logging =False ) ) return dataCollection
def collectDataForIndividualGraphics( self, startTime, endTime, types ): #find parameters """ @summary : Returns a list of ClientStatsPicklers instances, each of wich contains data for all the individual graphics. @return : List of ClientStatsPicklers instances. """ global _ dataCollection = [] for client in self.clientNames : # #Gather data from all previously created pickles.... if self.logger != None : try: self.logger.debug( _("Call to mergeHourlyPickles." ) ) self.logger.debug( _("Parameters used : %s %s %s") %( startTime, endTime, client ) ) except: pass if len( self.machines ) > 1 : clientArray = [] clientArray.append(client) statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\ clients = clientArray, fileType = self.fileType,\ machines = self.machines, groupName = self.groupName ) else:#only one machine, only merge different hours together statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = None , startTime = startTime, endTime = endTime,\ client = client, fileType = self.fileType, machine = self.machines[0] ) combinedMachineName = "" combinedMachineName = combinedMachineName.join( [ machine for machine in self.machines] ) dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\ statsCollection = statsCollection, machine = combinedMachineName,\ logger = None, logging =False ) ) return dataCollection
def collectDataForCombinedGraphics(self, startTime, endTime, types): """ @summary : Returns a list of one ClientStatsPicklers instance wich contains the combined data of all the individual graphics. @return : Returns a list of ONE ClientStatsPicklers """ dataCollection = [] statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\ clients = self.clientNames, fileType = self.fileType,\ machines = self.machines, groupName = self.groupName ) combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in self.machines]) #Verifier params utiliser par cette ligne dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\ statsCollection = statsCollection, machine = combinedMachineName,\ logger = None, logging = False ) ) return dataCollection
def collectDataForCombinedGraphics( self, startTime, endTime, types ): """ @summary : Returns a list of one ClientStatsPicklers instance wich contains the combined data of all the individual graphics. @return : Returns a list of ONE ClientStatsPicklers """ dataCollection = [] statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\ clients = self.clientNames, fileType = self.fileType,\ machines = self.machines, groupName = self.groupName ) combinedMachineName = "" combinedMachineName = combinedMachineName.join( [machine for machine in self.machines]) #Verifier params utiliser par cette ligne dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\ statsCollection = statsCollection, machine = combinedMachineName,\ logger = None, logging = False ) ) return dataCollection
def getTimeSeperatorsBasedOnAvailableMemory(startTime, endTime, clients, fileType, machines): """ @summary: returns the time seperators to be used for the transfer in a way that should prevent overloading memory. @param startTime: start time of the transfer to be attempted. @param endTime: end time of the transfer to be attempted. @param clients: lists of clients/sources to be transferred. @param fileType: tx or rx. @param machines: machines on wich the clients/sources reside. @return: the time seperators. """ width = 0 # Width in seconds of the transfer to be attempted seperators = [ ] # Time sperators representing every hour to be transferred. allFiles = [] # List of all pickle files that will be involved hourlyFiles = [] # List of all files to be handled for a certain hour. hourlyFileSizes = [ ] # Total file size of all the files to be handled at a certain hour. totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine. seperatorsBasedOnAvailableMemory = [ startTime, endTime ] # Suppose we have all the momory we need. width = (StatsDateLib.getSecondsSinceEpoch(endTime) - StatsDateLib.getSecondsSinceEpoch(startTime)) / StatsDateLib.HOUR seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime=startTime, width=width * StatsDateLib.HOUR, interval=StatsDateLib.HOUR)[:-1]) for seperator in seperators: hourlyFiles = PickleMerging.createNonMergedPicklesList( seperator, machines, fileType, clients) allFiles.extend(hourlyFiles) hourlyFileSizes.append( MemoryManagement.getTotalSizeListOfFiles(hourlyFiles)) totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles( allFiles) currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory( marginOfError=0.75 ) #never expect more than 25% of the avaiable memory to be avaiable for pickle loading. if totalSizeToloadInMemory >= currentlyAvailableMemory: seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentlyAvailableMemory, hourlyFileSizes) return seperatorsBasedOnAvailableMemory
def main(): """ @summary: Small test case to see if everything works fine """ statsConfig = StatsConfigParameters() statsConfig.getAllParameters() machineconfig = MachineConfigParameters() machineconfig.getParametersFromMachineConfigurationFile() currentTimeEpochFormat = time.time() - (120 * 60) endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat)) startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat - (StatsDateLib.DAY * 7))) print startTime, endTime groupName = statsConfig.groupParameters.groups[0] clients = statsConfig.groupParameters.groupsMembers[groupName] machines = statsConfig.groupParameters.groupsMachines[groupName] fileType = statsConfig.groupParameters.groupFileTypes[groupName] seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime=startTime, width=StatsDateLib.DAY * 7, interval=StatsDateLib.HOUR)[:-1]) listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators) listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles) currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555) if MemoryManagement.getTotalSizeListOfFiles( listOfFiles) > currentFreeMemory: seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes) print seperators else: print "We have %s bytes free and the pickles require %s bytes" % ( currentFreeMemory, getTotalSizeListOfFiles(listOfFiles)) print "we have enough memory to merge all these pickles."
def getTimeSeperatorsBasedOnAvailableMemory( startTime, endTime, clients, fileType, machines ): """ @summary: returns the time seperators to be used for the transfer in a way that should prevent overloading memory. @param startTime: start time of the transfer to be attempted. @param endTime: end time of the transfer to be attempted. @param clients: lists of clients/sources to be transferred. @param fileType: tx or rx. @param machines: machines on wich the clients/sources reside. @return: the time seperators. """ width = 0 # Width in seconds of the transfer to be attempted seperators = [] # Time sperators representing every hour to be transferred. allFiles =[] # List of all pickle files that will be involved hourlyFiles = [] # List of all files to be handled for a certain hour. hourlyFileSizes = [] # Total file size of all the files to be handled at a certain hour. totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine. seperatorsBasedOnAvailableMemory = [startTime, endTime] # Suppose we have all the momory we need. width = ( StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) ) / StatsDateLib.HOUR seperators = [ startTime ] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width= width*StatsDateLib.HOUR, interval=StatsDateLib.HOUR )[:-1]) for seperator in seperators: hourlyFiles = PickleMerging.createNonMergedPicklesList( seperator, machines, fileType, clients ) allFiles.extend( hourlyFiles ) hourlyFileSizes.append( MemoryManagement.getTotalSizeListOfFiles( hourlyFiles ) ) totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles( allFiles ) currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory( marginOfError = 0.75 )#never expect more than 25% of the avaiable memory to be avaiable for pickle loading. if totalSizeToloadInMemory >= currentlyAvailableMemory: seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentlyAvailableMemory, hourlyFileSizes ) return seperatorsBasedOnAvailableMemory
def main(): """ @summary: Small test case to see if everything works fine """ statsConfig = StatsConfigParameters() statsConfig.getAllParameters() machineconfig = MachineConfigParameters() machineconfig.getParametersFromMachineConfigurationFile() currentTimeEpochFormat = time.time() -(120*60) endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) ) startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) ) print startTime, endTime groupName = statsConfig.groupParameters.groups[0] clients = statsConfig.groupParameters.groupsMembers[ groupName ] machines = statsConfig.groupParameters.groupsMachines[ groupName ] fileType = statsConfig.groupParameters.groupFileTypes[ groupName ] seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1]) listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators ) listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles) currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555) if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory: seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes ) print seperators else: print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) ) print "we have enough memory to merge all these pickles."