def getMergedData( clients, fileType, machines, startTime, endTime, groupName = "", logger = None ):
"""
This method returns all data comprised between startTime and endTime as
to be able to build pairs.
"""
if fileType == "tx":
types = [ "errors","bytecount","latency", ]
else:
types = [ "errors","bytecount" ]
#print startTime, endTime
if len( machines ) > 1 or len( clients) > 1:
statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = logger , startTime = startTime, endTime = endTime, clients = clients, fileType = fileType, machines = machines, groupName = groupName )
else:#only one machine, only merge different hours together
statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = logger , startTime = startTime, endTime = endTime, client = clients[0], fileType = fileType, machine = machines[0] )
combinedMachineName = ""
for machine in machines:
combinedMachineName = combinedMachineName + machine
dataCollector = StatsPickler( client = clients[0], statsTypes = types, directory = "", statsCollection = statsCollection, machine = combinedMachineName, logger = logger )
return dataCollector
def getMergedData(clients,
fileType,
machines,
startTime,
endTime,
groupName="",
logger=None):
"""
This method returns all data comprised between startTime and endTime as
to be able to build pairs.
"""
if fileType == "tx":
types = [
"errors",
"bytecount",
"latency",
]
else:
types = ["errors", "bytecount"]
#print startTime, endTime
if len(machines) > 1 or len(clients) > 1:
statsCollection = PickleMerging.mergePicklesFromDifferentSources(
logger=logger,
startTime=startTime,
endTime=endTime,
clients=clients,
fileType=fileType,
machines=machines,
groupName=groupName)
else: #only one machine, only merge different hours together
statsCollection = PickleMerging.mergePicklesFromDifferentHours(
logger=logger,
startTime=startTime,
endTime=endTime,
client=clients[0],
fileType=fileType,
machine=machines[0])
combinedMachineName = ""
for machine in machines:
combinedMachineName = combinedMachineName + machine
dataCollector = StatsPickler(client=clients[0],
statsTypes=types,
directory="",
statsCollection=statsCollection,
machine=combinedMachineName,
logger=logger)
return dataCollector
def collectDataForIndividualGraphics(self, startTime, endTime, types):
#find parameters
"""
@summary : Returns a list of ClientStatsPicklers
instances, each of wich contains data
for all the individual graphics.
@return : List of ClientStatsPicklers instances.
"""
global _
dataCollection = []
for client in self.clientNames: #
#Gather data from all previously created pickles....
if self.logger != None:
try:
self.logger.debug(_("Call to mergeHourlyPickles."))
self.logger.debug(
_("Parameters used : %s %s %s") %
(startTime, endTime, client))
except:
pass
if len(self.machines) > 1:
clientArray = []
clientArray.append(client)
statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\
clients = clientArray, fileType = self.fileType,\
machines = self.machines, groupName = self.groupName )
else: #only one machine, only merge different hours together
statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = None , startTime = startTime, endTime = endTime,\
client = client, fileType = self.fileType, machine = self.machines[0] )
combinedMachineName = ""
combinedMachineName = combinedMachineName.join(
[machine for machine in self.machines])
dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\
statsCollection = statsCollection, machine = combinedMachineName,\
logger = None, logging =False ) )
return dataCollection
def collectDataForIndividualGraphics( self, startTime, endTime, types ):
#find parameters
"""
@summary : Returns a list of ClientStatsPicklers
instances, each of wich contains data
for all the individual graphics.
@return : List of ClientStatsPicklers instances.
"""
global _
dataCollection = []
for client in self.clientNames : #
#Gather data from all previously created pickles....
if self.logger != None :
try:
self.logger.debug( _("Call to mergeHourlyPickles." ) )
self.logger.debug( _("Parameters used : %s %s %s") %( startTime, endTime, client ) )
except:
pass
if len( self.machines ) > 1 :
clientArray = []
clientArray.append(client)
statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\
clients = clientArray, fileType = self.fileType,\
machines = self.machines, groupName = self.groupName )
else:#only one machine, only merge different hours together
statsCollection = PickleMerging.mergePicklesFromDifferentHours( logger = None , startTime = startTime, endTime = endTime,\
client = client, fileType = self.fileType, machine = self.machines[0] )
combinedMachineName = ""
combinedMachineName = combinedMachineName.join( [ machine for machine in self.machines] )
dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\
statsCollection = statsCollection, machine = combinedMachineName,\
logger = None, logging =False ) )
return dataCollection
def collectDataForCombinedGraphics(self, startTime, endTime, types):
"""
@summary : Returns a list of one ClientStatsPicklers
instance wich contains the combined data
of all the individual graphics.
@return : Returns a list of ONE ClientStatsPicklers
"""
dataCollection = []
statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\
clients = self.clientNames, fileType = self.fileType,\
machines = self.machines, groupName = self.groupName )
combinedMachineName = ""
combinedMachineName = combinedMachineName.join(
[machine for machine in self.machines])
#Verifier params utiliser par cette ligne
dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\
statsCollection = statsCollection, machine = combinedMachineName,\
logger = None, logging = False ) )
return dataCollection
def collectDataForCombinedGraphics( self, startTime, endTime, types ):
"""
@summary : Returns a list of one ClientStatsPicklers
instance wich contains the combined data
of all the individual graphics.
@return : Returns a list of ONE ClientStatsPicklers
"""
dataCollection = []
statsCollection = PickleMerging.mergePicklesFromDifferentSources( logger = None , startTime = startTime, endTime = endTime,\
clients = self.clientNames, fileType = self.fileType,\
machines = self.machines, groupName = self.groupName )
combinedMachineName = ""
combinedMachineName = combinedMachineName.join( [machine for machine in self.machines])
#Verifier params utiliser par cette ligne
dataCollection.append( StatsPickler( client = self.clientNames, statsTypes = types, directory = self.directory,\
statsCollection = statsCollection, machine = combinedMachineName,\
logger = None, logging = False ) )
return dataCollection
def getTimeSeperatorsBasedOnAvailableMemory(startTime, endTime, clients,
fileType, machines):
"""
@summary: returns the time seperators to be used for the transfer
in a way that should prevent overloading memory.
@param startTime: start time of the transfer to be attempted.
@param endTime: end time of the transfer to be attempted.
@param clients: lists of clients/sources to be transferred.
@param fileType: tx or rx.
@param machines: machines on wich the clients/sources reside.
@return: the time seperators.
"""
width = 0 # Width in seconds of the transfer to be attempted
seperators = [
] # Time sperators representing every hour to be transferred.
allFiles = [] # List of all pickle files that will be involved
hourlyFiles = [] # List of all files to be handled for a certain hour.
hourlyFileSizes = [
] # Total file size of all the files to be handled at a certain hour.
totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory
currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine.
seperatorsBasedOnAvailableMemory = [
startTime, endTime
] # Suppose we have all the momory we need.
width = (StatsDateLib.getSecondsSinceEpoch(endTime) -
StatsDateLib.getSecondsSinceEpoch(startTime)) / StatsDateLib.HOUR
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=width * StatsDateLib.HOUR,
interval=StatsDateLib.HOUR)[:-1])
for seperator in seperators:
hourlyFiles = PickleMerging.createNonMergedPicklesList(
seperator, machines, fileType, clients)
allFiles.extend(hourlyFiles)
hourlyFileSizes.append(
MemoryManagement.getTotalSizeListOfFiles(hourlyFiles))
totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles(
allFiles)
currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory(
marginOfError=0.75
) #never expect more than 25% of the avaiable memory to be avaiable for pickle loading.
if totalSizeToloadInMemory >= currentlyAvailableMemory:
seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentlyAvailableMemory, hourlyFileSizes)
return seperatorsBasedOnAvailableMemory
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() - (120 * 60)
endTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat))
startTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat -
(StatsDateLib.DAY * 7)))
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[groupName]
machines = statsConfig.groupParameters.groupsMachines[groupName]
fileType = statsConfig.groupParameters.groupFileTypes[groupName]
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=StatsDateLib.DAY * 7,
interval=StatsDateLib.HOUR)[:-1])
listOfFiles = PickleMerging.createMergedPicklesList(
startTime, endTime, clients, groupName, fileType, machines, seperators)
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles(
listOfFiles) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentFreeMemory, listOfFileSizes)
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" % (
currentFreeMemory, getTotalSizeListOfFiles(listOfFiles))
print "we have enough memory to merge all these pickles."
def getTimeSeperatorsBasedOnAvailableMemory( startTime, endTime, clients, fileType, machines ):
"""
@summary: returns the time seperators to be used for the transfer
in a way that should prevent overloading memory.
@param startTime: start time of the transfer to be attempted.
@param endTime: end time of the transfer to be attempted.
@param clients: lists of clients/sources to be transferred.
@param fileType: tx or rx.
@param machines: machines on wich the clients/sources reside.
@return: the time seperators.
"""
width = 0 # Width in seconds of the transfer to be attempted
seperators = [] # Time sperators representing every hour to be transferred.
allFiles =[] # List of all pickle files that will be involved
hourlyFiles = [] # List of all files to be handled for a certain hour.
hourlyFileSizes = [] # Total file size of all the files to be handled at a certain hour.
totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory
currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine.
seperatorsBasedOnAvailableMemory = [startTime, endTime] # Suppose we have all the momory we need.
width = ( StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) ) / StatsDateLib.HOUR
seperators = [ startTime ]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width= width*StatsDateLib.HOUR, interval=StatsDateLib.HOUR )[:-1])
for seperator in seperators:
hourlyFiles = PickleMerging.createNonMergedPicklesList( seperator, machines, fileType, clients )
allFiles.extend( hourlyFiles )
hourlyFileSizes.append( MemoryManagement.getTotalSizeListOfFiles( hourlyFiles ) )
totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles( allFiles )
currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory( marginOfError = 0.75 )#never expect more than 25% of the avaiable memory to be avaiable for pickle loading.
if totalSizeToloadInMemory >= currentlyAvailableMemory:
seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentlyAvailableMemory, hourlyFileSizes )
return seperatorsBasedOnAvailableMemory
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() -(120*60)
endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) )
startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) )
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[ groupName ]
machines = statsConfig.groupParameters.groupsMachines[ groupName ]
fileType = statsConfig.groupParameters.groupFileTypes[ groupName ]
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1])
listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators )
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes )
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) )
print "we have enough memory to merge all these pickles."
