def saveList( self, user, clients ):
"""
@summary : Saves list.
@note : Will include modification made in updateFileInlist method
@param clients : Client to wich the file is related(used to narrow down searchs)
@param user : Name of the client, person, etc.. wich has a relation with the
file.
"""
statsPaths = StatsPaths()
statsPaths.setPaths()
directory = statsPaths.STATSDATA + "fileAccessVersions/"
combinedName = ""
for client in clients:
combinedName = combinedName + client
fileName = combinedName + "_" + user
if not os.path.isdir( directory ):
os.makedirs( directory, mode=0777 )
#create directory
completeFilename = directory + fileName
#print "saving %s" %completeFilename
CpickleWrapper.save( object = self.savedFileList, filename = completeFilename )
def saveList(self, user, clients):
"""
@summary : Saves list.
@note : Will include modification made in updateFileInlist method
@param clients : Client to wich the file is related(used to narrow down searchs)
@param user : Name of the client, person, etc.. wich has a relation with the
file.
"""
statsPaths = StatsPaths()
statsPaths.setPaths()
directory = statsPaths.STATSDATA + "fileAccessVersions/"
combinedName = ""
for client in clients:
combinedName = combinedName + client
fileName = combinedName + "_" + user
if not os.path.isdir(directory):
os.makedirs(directory, mode=0777)
#create directory
completeFilename = directory + fileName
#print "saving %s" %completeFilename
CpickleWrapper.save(object=self.savedFileList,
filename=completeFilename)
def addAutomaticUpdateToLogs( self, timeOfUpdateInIsoFormat, currentUpdateFrequency = None ):
"""
@summary : Writes a new file in the log folder containing
the current update frequency.
@param timeOfUpdateInIsoFormat: Time that the entries name will sport.
"""
paths = StatsPaths()
paths.setPaths()
fileName = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + str( timeOfUpdateInIsoFormat ).replace( " ", "_" )
#Safety to make sure
if not os.path.isdir( os.path.dirname( fileName ) ):
os.makedirs( os.path.dirname( fileName ), 0777 )
if currentUpdateFrequency == None :
currentUpdateFrequency = self.getCurrentUpdateFrequency()
CpickleWrapper.save( currentUpdateFrequency, fileName )
allEntries = os.listdir(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/")
allEntries.sort()
entriesToRemove = allEntries[ :-self.numberOfLogsToKeep]
for entrytoRemove in entriesToRemove:
os.remove(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + entrytoRemove )
def saveAccessDictionary( self ):
"""
@summary: Saves the current accessDictionary into the
accessfile.
"""
if not os.path.isdir( os.path.dirname( self.accessFile ) ):
os.makedirs( os.path.dirname( self.accessFile ) )
CpickleWrapper.save( self.accessDictionary, self.accessFile )
def saveAccessDictionary(self):
"""
@summary: Saves the current accessDictionary into the
accessfile.
"""
if not os.path.isdir(os.path.dirname(self.accessFile)):
os.makedirs(os.path.dirname(self.accessFile))
CpickleWrapper.save(self.accessDictionary, self.accessFile)
def saveCurrentMachineParameters( machineParameters ):
"""
@summary : Saves the current machineParameters into
the /data/previousMachineParameters file.
@param machineParameters: Machine parameters to save.
"""
paths = StatsPaths()
paths.setPaths()
if not os.path.isdir( os.path.dirname( paths.STATSPREVIOUSMACHINEPARAMS ) ):
os.makedirs( os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS) )
CpickleWrapper.save( machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def saveCurrentMachineParameters(machineParameters):
"""
@summary : Saves the current machineParameters into
the /data/previousMachineParameters file.
@param machineParameters: Machine parameters to save.
"""
paths = StatsPaths()
paths.setPaths()
if not os.path.isdir(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)):
os.makedirs(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS))
CpickleWrapper.save(machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def loadAccessFile(self):
"""
@summary: Loads the accessFile into the accessDictionary.
"""
self.accessDictionary = CpickleWrapper.load(self.accessFile)
def getSavedList( self, user, clients ):
"""
@summary : Returns the checksum of the files contained in the saved list.
"""
self.savedFileList = {}
statsPaths = StatsPaths()
statsPaths.setPaths()
directory = statsPaths.STATSDATA + "fileAccessVersions/"
combinedName = ""
for client in clients:
combinedName = combinedName + client
fileName = combinedName + "_" + user
try :
self.savedFileList = CpickleWrapper.load( directory + fileName )
if self.savedFileLis == None :
self.savedFileList = {}
except: # if file does not exist
pass
return self.savedFileList
def getSavedList(self, user, clients):
"""
@summary : Returns the checksum of the files contained in the saved list.
"""
self.savedFileList = {}
statsPaths = StatsPaths()
statsPaths.setPaths()
directory = statsPaths.STATSDATA + "fileAccessVersions/"
combinedName = ""
for client in clients:
combinedName = combinedName + client
fileName = combinedName + "_" + user
try:
self.savedFileList = CpickleWrapper.load(directory + fileName)
if self.savedFileLis == None:
self.savedFileList = {}
except: # if file does not exist
pass
return self.savedFileList
def loadAccessFile(self):
"""
@summary: Loads the accessFile into the accessDictionary.
"""
self.accessDictionary = CpickleWrapper.load( self.accessFile )
def printPickle(pickle, outputFile=""):
"""
Print content of a pickle file containing a FileStatscollector
instance on the desired output.
Default output is the console screen.
File can be specified to make reading easier.
"""
if outputFile != "":
fileHandle = open(outputFile, 'w')
old_stdout = sys.stdout
sys.stdout = fileHandle
statsCollection = CpickleWrapper.load(pickle)
print _("Pickle used : %s") % pickle
print _("\n\nFiles used : %s") % statsCollection.files
print _("Starting date: %s") % statsCollection.startTime
print _("Interval: %s") % statsCollection.interval
print _("End time : %s") % statsCollection.endTime
print _("nbEntries : %s") % statsCollection.nbEntries
for j in range(statsCollection.nbEntries):
print _("\nEntry's interval : %s - %s ") % (
statsCollection.fileEntries[j].startTime,
statsCollection.fileEntries[j].endTime)
print _("Files : ")
print statsCollection.fileEntries[j].files
print _("Products : ")
print statsCollection.fileEntries[j].values.productTypes
print _("Values :")
print statsCollection.fileEntries[j].values.dictionary
print _("Means :")
print statsCollection.fileEntries[j].means
print _("Medians")
print statsCollection.fileEntries[j].medians
print _("Minimums")
print statsCollection.fileEntries[j].minimums
print _("Maximums")
print statsCollection.fileEntries[j].maximums
print _("Time where max occured :")
print statsCollection.fileEntries[j].timesWhereMaxOccured
print _("Total")
print statsCollection.fileEntries[j].totals
print _("Files over maximum latency")
print statsCollection.fileEntries[j].filesOverMaxLatency
if outputFile != "":
fileHandle.close()
sys.stdout = old_stdout #resets standard output
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\
endTime = "2006-07-31 19:00:00", client = "satnet",\
machine = "pdsPM", fileType = "tx" ):
"""
@summary : This method merges entire hourly pickles files together.
@None : This does not support merging part of the data of pickles.
"""
if logger != None :
logger.debug( _("Call to mergeHourlyPickles received.") )
logging = True
else:
logging = False
pickles = []
entries = {}
width = StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime )
startTime = StatsDateLib.getIsoWithRoundedHours( startTime )
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=width, interval=60*StatsDateLib.MINUTE )[:-1])
for seperator in seperators :
pickles.append( StatsPickler.buildThisHoursFileName( client = client, offset = 0, currentTime = seperator, machine = machine, fileType = fileType ) )
startingNumberOfEntries = 0
#print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
for pickle in pickles :
if os.path.isfile( pickle ) :
tempCollection = CpickleWrapper.load( pickle )
if tempCollection != None :
for i in xrange( len( tempCollection.fileEntries ) ):
entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i]
startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries )
else:
sys.exit()
else:
emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60, entries = {} )
for i in xrange( 60 ):
entries[i + startingNumberOfEntries ] = emptyEntries [i]
startingNumberOfEntries = startingNumberOfEntries + 60
#print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
statsCollection = FileStatsCollector( startTime = startTime , endTime = endTime, interval = StatsDateLib.MINUTE, totalWidth = width, fileEntries = entries,fileType= fileType, logger = logger, logging = logging )
return statsCollection
def printPickle( pickle, outputFile = "" ):
"""
Print content of a pickle file containing a FileStatscollector
instance on the desired output.
Default output is the console screen.
File can be specified to make reading easier.
"""
if outputFile != "":
fileHandle = open( outputFile , 'w' )
old_stdout = sys.stdout
sys.stdout = fileHandle
statsCollection = CpickleWrapper.load( pickle )
print _("Pickle used : %s" )%pickle
print _("\n\nFiles used : %s" ) %statsCollection.files
print _("Starting date: %s" ) % statsCollection.startTime
print _("Interval: %s" ) %statsCollection.interval
print _("End time : %s" ) %statsCollection.endTime
print _("nbEntries : %s" ) %statsCollection.nbEntries
for j in range( statsCollection.nbEntries ):
print _("\nEntry's interval : %s - %s " ) %( statsCollection.fileEntries[j].startTime, statsCollection.fileEntries[j].endTime )
print _("Files : " )
print statsCollection.fileEntries[j].files
print _("Products : " )
print statsCollection.fileEntries[j].values.productTypes
print _("Values :" )
print statsCollection.fileEntries[j].values.dictionary
print _("Means :" )
print statsCollection.fileEntries[j].means
print _("Medians" )
print statsCollection.fileEntries[j].medians
print _("Minimums" )
print statsCollection.fileEntries[j].minimums
print _("Maximums" )
print statsCollection.fileEntries[j].maximums
print _("Time where max occured :" )
print statsCollection.fileEntries[j].timesWhereMaxOccured
print _("Total" )
print statsCollection.fileEntries[j].totals
print _("Files over maximum latency" )
print statsCollection.fileEntries[j].filesOverMaxLatency
if outputFile != "":
fileHandle.close()
sys.stdout = old_stdout #resets standard output
def changeInUpdateFrenquencyFoundDuringTimespan( self, startTime, endTime ):
"""
@summary : Searchs whether or not there was a change during the specified timespan.
@param statTime : Start time in the iso format of the time span to survey.
@param endTime : End time in the iso format of the time span to survey/
@return : True or false whether there was a change or not, plus the original
frequency and the new frequency.
"""
changeWasMade = False
paths = StatsPaths()
paths.setPaths()
updatesDoneDuringTimespan = self.__getAutomaticUpdatesDoneDuringTimeSpan( startTime, endTime )
updatesDoneDuringTimespan.sort()
if updatesDoneDuringTimespan != []:
fileName = paths.STATSTEMPAUTUPDTLOGS + str(updatesDoneDuringTimespan[0]).replace( " ", "_" )
originalUpdateFrequency = CpickleWrapper.load(fileName)
newUpdateFrequency = originalUpdateFrequency
for update in updatesDoneDuringTimespan:
fileName = paths.STATSTEMPAUTUPDTLOGS + str(update).replace( " ", "_" )
newUpdateFrequency = CpickleWrapper.load(fileName)
if newUpdateFrequency != originalUpdateFrequency:
changeWasMade = True
break
return changeWasMade, originalUpdateFrequency, newUpdateFrequency
def getMachineParametersFromPreviousCall() :
"""
@summary: Gets the machine parameters that are
saved in data/previousMachineParameters.
@return: Returns the saved machine parameters.
"""
paths = StatsPaths()
paths.setPaths()
previousMachineParams = None
if os.path.isfile( paths.STATSPREVIOUSMACHINEPARAMS ):
previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS )
return previousMachineParams
def getMachineParametersFromPreviousCall():
"""
@summary: Gets the machine parameters that are
saved in data/previousMachineParameters.
@return: Returns the saved machine parameters.
"""
paths = StatsPaths()
paths.setPaths()
previousMachineParams = None
if os.path.isfile(paths.STATSPREVIOUSMACHINEPARAMS):
previousMachineParams = CpickleWrapper.load(
paths.STATSPREVIOUSMACHINEPARAMS)
return previousMachineParams
def previousUpdateFrequency(self):
"""
@summary : Finds and returns the frequency
of the previous update.
@return : The freqency of the previous update.
"""
paths = StatsPaths()
paths.setPaths()
lastUpdate = self.getTimeOfLastUpdateInLogs()
fileName = paths.STATSTEMPAUTUPDTLOGS + str(lastUpdate).replace( " ", "_" )
lastUpdateFrequency = CpickleWrapper.load(fileName)
return lastUpdateFrequency
def main():
"""
Test cases. These test must work.
Please run the tests after modifying this
file to make sure tests still work.
"""
import commands
##################################################################################################
#
# This section test the utility methods which do not require log files
#
##################################################################################################
updateManager = AutomaticUpdatesManager( 10, "pxStatsStartup" )
print ""
print ""
print "getMissingDaysBetweenUpdates test #1 : "
print ""
print """updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) : """
print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] ")
print "Obtained result : %s " %updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" )
if not updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) ==\
['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.")
print ""
print ""
print "getMissingWeeksBetweenUpdates test #1 : "
print ""
print """updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) : """
print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00']")
print "Obtained result : %s " %updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" )
if not updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) ==\
['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.")
print ""
print ""
print "getMissingMonthssBetweenUpdates test #1 : "
print ""
print """updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) : """
print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00','2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ")
print "Obtained result : %s " %updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" )
if not (updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00', '2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ) : raise AssertionError("getMissingMonthsBetweenUpdates test #1 is broken.")
print ""
print ""
print "getMissingYearsBetweenUpdates test #1 : "
print ""
print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """
print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']")
print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" )
if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.")
print ""
print ""
print "getMissingYearsBetweenUpdates test #1 : "
print ""
print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """
print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']")
print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" )
if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.")
print ""
print ""
print "getCurrentUpdateFrequency() test #1"
print ""
print """ updateManager.getCurrentUpdateFrequency() """
print "PLEASE VERIFY THAT RESULT MATCH UPDATE FREQUENCY FOUND IN CRONTAB ENTRY."
print "Type crontab -l to see current crontab configuration."
print "Obtained result : %s" %( updateManager.getCurrentUpdateFrequency() )
print ""
##################################################################################################
#
# This section requires log files. If log files currently exist, will temporarily copy
# them to a temp folder. Otherwise, will create some test updates files usefull only for
# testing.
#
##################################################################################################
paths = StatsPaths( )
paths.setPaths()
print paths.STATSTEMPAUTUPDTLOGS
if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS ) :
print commands.getoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS, paths.STATSTEMPAUTUPDTLOGS[:-1] + ".old" ) )
os.makedirs( paths.STATSTEMPAUTUPDTLOGS, 0777 )
print ""
print ""
print "getTimeOfLastUpdateInLogs test#1"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs"""
print """updateManager.getTimeOfLastUpdateInLogs() """
updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" )
updateManager.getTimeOfLastUpdateInLogs()
print "Expected results : 2008-02-01 04:00:00"
print "Obtained results : %s" %updateManager.getTimeOfLastUpdateInLogs( )
if not updateManager.getTimeOfLastUpdateInLogs( ) == "2008-02-01 04:00:00" : raise AssertionError( "getTimeOfLastUpdateInLogs test#1 is broken" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" )
print ""
print ""
print "isFirstUpdateOfTheDay test#1( valid case )"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" ) : Create an update for that day """
print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day"""
updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" )
print "Expected result : True"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ))
if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-14_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheDay test#2(invalid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" ) : Create an update for that day """
print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day"""
updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" )
print "Expected result : False"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ))
if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-15_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheWeek test#1(valid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" ) : Create an update for that week """
print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week"""
updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" )
print "Expected result : True"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ))
if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-20_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheWeek test#2(invalid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that week """
print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week"""
updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" )
print "Expected result : False"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ))
if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheMonth test#1(valid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" ) : Create an update for that Month """
print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month"""
updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" )
print "Expected result : True"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) )
if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-01-20_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheMonth test#2(invalid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Month """
print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month"""
updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" )
print "Expected result : False"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ))
if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheYear test#1(valid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" ) : Create an update for that Year """
print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year"""
updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" )
print "Expected result : True"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ))
if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheYear test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-01-20_23:00:00" )
print ""
print ""
print "isFirstUpdateOfTheYear test#2(invalid case)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Year """
print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year"""
updateManager.addAutomaticUpdateToLogs( "2008-02-27_23:00:00" )
print "Expected result : False"
print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ))
if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken")
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" )
print ""
print ""
print "getTimeSinceLastUpdate test#1"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update to test with"""
print """updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) : Date to test with """
updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" )
print "Expected result : "
print "Obtained result : %s" %( updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) )
if not updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) == 86400 : raise AssertionError( "getTimeSinceLastUpdate test#1 is broken" )
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" )
print ""
print ""
print "getNbAutomaticUpdatesDoneDuringTimeSpan test#1"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs"""
print """updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-01-01 00:50:00","2008-02-01 03:30:00") """
updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" )
print "Expected results : 3"
print "Obtained results : %s" %updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00")
if not updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00") == 3 : raise AssertionError( "getNbAutomaticUpdatesDoneDuringTimeSpan test#1 is broken" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" )
print ""
print ""
print "changeInUpdateFrenquencyFoundDuringTimespan test#1(no change to be detected)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs"""
print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")"""
updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" )
print "Expected results : False"
print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")
if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == (False, updateManager.getCurrentUpdateFrequency(), updateManager.getCurrentUpdateFrequency()) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" )
print ""
print ""
print "changeInUpdateFrenquencyFoundDuringTimespan test#2(Frequency change to be detected)"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs"""
print """CpickleWrapper.save(0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00") : Corrupt an entry for testing purposes"""
print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")"""
updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" )
CpickleWrapper.save(-1, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00")
print "Expected results : True %s -1" %updateManager.getCurrentUpdateFrequency()
print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-05 05:00:00")
if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == ( True, updateManager.getCurrentUpdateFrequency(), -1 ) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" )
os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" )
print ""
print ""
print "previousUpdateFrequency test#1"
print ""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs"""
print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs"""
print """updateManager.previousUpdateFrequency() """
updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" )
updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" )
CpickleWrapper.save( 0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
print "Expected results : 0 "
print "Obtained results : %s" %updateManager.previousUpdateFrequency()
if not updateManager.previousUpdateFrequency() == 0 : raise AssertionError( "previousUpdateFrequency test#1 is broken" )
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" )
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" )
os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" )
os.removedirs( paths.STATSTEMPAUTUPDTLOGS )
if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS + ".old" ) :
commands.getstatusoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS + ".old", paths.STATSTEMPAUTUPDTLOGS ) )
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\
clientName = "" , combinedMachineName = "", currentTime = "",\
fileType = "tx" ):
"""
@summary: This methods receives a list of filenames referring to pickled FileStatsEntries.
After the merger pickles get saved since they might be reused somewhere else.
@precondition: Pickle should be of the same timespan and bucket width.
If not no merging will occur.
"""
if logger != None:
logger.debug(_("Call to mergePickles received."))
logging = True
else:
logging = False
entryList = []
for pickle in pickleNames: #for every pickle we eneed to merge
if os.path.isfile(pickle):
entryList.append(CpickleWrapper.load(pickle))
else: #Use empty entry if there is no existing pickle of that name
endTime = StatsDateLib.getIsoFromEpoch(
StatsDateLib.getSecondsSinceEpoch(currentTime) +
StatsDateLib.HOUR)
entryList.append(
FileStatsCollector(startTime=currentTime,
endTime=endTime,
logger=logger,
logging=logging))
if logger != None:
logger.warning(
_("Pickle named %s did not exist. Empty entry was used instead."
) % pickle)
#start off with a carbon copy of first pickle in list.
newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\
endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\
firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\
maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\
logging = logging )
if PickleMerging.entryListIsValid(entryList) == True:
for i in range(1, len(entryList)): #add other entries
for file in entryList[i].files:
if file not in newFSC.files:
newFSC.files.append(file)
for j in range(len(newFSC.fileEntries)): # add all entries
newFSC.fileEntries[j].values.productTypes.extend(
entryList[i].fileEntries[j].values.productTypes)
newFSC.fileEntries[j].files.extend(
entryList[i].fileEntries[j].files)
newFSC.fileEntries[j].times.extend(
entryList[i].fileEntries[j].times)
newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[
j].nbFiles + (newFSC.fileEntries[j].nbFiles)
for type in newFSC.statsTypes:
newFSC.fileEntries[j].values.dictionary[type].extend(
entryList[i].fileEntries[j].values.dictionary[type]
)
newFSC.fileEntries[j].values.rows = newFSC.fileEntries[
j].values.rows + entryList[i].fileEntries[j].values.rows
newFSC = newFSC.setMinMaxMeanMedians(
startingBucket=0, finishingBucket=newFSC.nbEntries - 1)
else: #Did not merge pickles named. Pickle list was not valid."
if logger != None:
logger.warning(
_("Did not merge pickles named : %s. Pickle list was not valid."
) % pickleNames)
logger.warning(
_("Filled with empty entries instead.") % pickleNames)
newFSC.fileEntries = PickleMerging.fillWithEmptyEntries(
nbEmptyEntries=60, entries={})
#prevents us from having ro remerge file later on.
temp = newFSC.logger
del newFSC.logger
CpickleWrapper.save(newFSC, mergedPickleName)
try:
os.chmod(mergedPickleName, 0777)
except:
pass
#print "saved :%s" %mergedPickleName
newFSC.logger = temp
return newFSC
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\
endTime = "2006-07-31 19:00:00", client = "satnet",\
machine = "pdsPM", fileType = "tx" ):
"""
@summary : This method merges entire hourly pickles files together.
@None : This does not support merging part of the data of pickles.
"""
if logger != None:
logger.debug(_("Call to mergeHourlyPickles received."))
logging = True
else:
logging = False
pickles = []
entries = {}
width = StatsDateLib.getSecondsSinceEpoch(
endTime) - StatsDateLib.getSecondsSinceEpoch(startTime)
startTime = StatsDateLib.getIsoWithRoundedHours(startTime)
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(startTime=startTime,
width=width,
interval=60 *
StatsDateLib.MINUTE)[:-1])
for seperator in seperators:
pickles.append(
StatsPickler.buildThisHoursFileName(client=client,
offset=0,
currentTime=seperator,
machine=machine,
fileType=fileType))
startingNumberOfEntries = 0
#print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
for pickle in pickles:
if os.path.isfile(pickle):
tempCollection = CpickleWrapper.load(pickle)
if tempCollection != None:
for i in xrange(len(tempCollection.fileEntries)):
entries[startingNumberOfEntries +
i] = tempCollection.fileEntries[i]
startingNumberOfEntries = startingNumberOfEntries + len(
tempCollection.fileEntries)
else:
sys.exit()
else:
emptyEntries = PickleMerging.fillWithEmptyEntries(
nbEmptyEntries=60, entries={})
for i in xrange(60):
entries[i + startingNumberOfEntries] = emptyEntries[i]
startingNumberOfEntries = startingNumberOfEntries + 60
#print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
statsCollection = FileStatsCollector(startTime=startTime,
endTime=endTime,
interval=StatsDateLib.MINUTE,
totalWidth=width,
fileEntries=entries,
fileType=fileType,
logger=logger,
logging=logging)
return statsCollection
"""
small test case. Tests if everything works plus gives an idea on proper usage.
"""
import time
statsPaths = StatsPaths()
statsPaths.setPaths()
timeA= time.time()
types = [ "latency", "errors","bytecount" ]
filename = statsPaths.STATSLOGS + 'someMachine/tx_someclient.log'
startingHours=["00:00:00","01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00" ]
endingHours = ["01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00", "00:00:00" ]
stats = FileStatsCollector( files = [ filename ], statsTypes = types , startTime = '2007-06-12 %s' %startingHours[2], endTime = '2007-06-12 %s' %endingHours[10], interval = 1*MINUTE, totalWidth = 9*HOUR )
stats.collectStats()
timeB = time.time()
print timeB - timeA
saveFile = STATSPATHS.STATSDATA + "testNew"
del stats.logger
CpickleWrapper.save( object = stats, filename = saveFile )
def collectStats(self,
types,
directory,
fileType="tx",
startTime='2006-05-18 00:00:00',
endTime="",
interval=60 * StatsDateLib.MINUTE,
save=True):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger #Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None: #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile(self.pickleName):
if self.logger != None:
self.logger.warning(
_("User tried to modify allready filled pickle file."))
self.logger.warning(
_("Pickle was named : %s") % self.pickleName)
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName(
client=self.client,
currentTime=startTime,
machine=self.machine,
fileType=fileType)
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats(endTime)
self.logger = temp
if save == True: # must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save(object=self.statsCollection,
filename=self.pickleName)
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname(self.pickleName)
while (dirname != STATSPATHS.STATSPICKLES[:-1]
): #[:-1] removes the last / character
try:
os.chmod(dirname, 0777)
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced:
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info(
_("Saved pickle named : %s ") % self.pickleName)
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\
clientName = "" , combinedMachineName = "", currentTime = "",\
fileType = "tx" ):
"""
@summary: This methods receives a list of filenames referring to pickled FileStatsEntries.
After the merger pickles get saved since they might be reused somewhere else.
@precondition: Pickle should be of the same timespan and bucket width.
If not no merging will occur.
"""
if logger != None :
logger.debug( _("Call to mergePickles received.") )
logging = True
else:
logging = False
entryList = []
for pickle in pickleNames:#for every pickle we eneed to merge
if os.path.isfile( pickle ):
entryList.append( CpickleWrapper.load( pickle ) )
else:#Use empty entry if there is no existing pickle of that name
endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( currentTime ) + StatsDateLib.HOUR )
entryList.append( FileStatsCollector( startTime = currentTime, endTime = endTime,logger =logger, logging =logging ) )
if logger != None :
logger.warning( _("Pickle named %s did not exist. Empty entry was used instead.") %pickle )
#start off with a carbon copy of first pickle in list.
newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\
endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\
firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\
maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\
logging = logging )
if PickleMerging.entryListIsValid( entryList ) == True :
for i in range ( 1 , len( entryList ) ): #add other entries
for file in entryList[i].files :
if file not in newFSC.files :
newFSC.files.append( file )
for j in range( len( newFSC.fileEntries ) ) : # add all entries
newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes )
newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files )
newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times )
newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[j].nbFiles + ( newFSC.fileEntries[ j ].nbFiles)
for type in newFSC.statsTypes :
newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] )
newFSC.fileEntries[j].values.rows = newFSC.fileEntries[j].values.rows + entryList[i].fileEntries[j].values.rows
newFSC = newFSC.setMinMaxMeanMedians( startingBucket = 0 , finishingBucket = newFSC.nbEntries -1 )
else:#Did not merge pickles named. Pickle list was not valid."
if logger != None :
logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid.") %pickleNames )
logger.warning( _("Filled with empty entries instead.") %pickleNames )
newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60 , entries = {} )
#prevents us from having ro remerge file later on.
temp = newFSC.logger
del newFSC.logger
CpickleWrapper.save( newFSC, mergedPickleName )
try:
os.chmod( mergedPickleName, 0777 )
except:
pass
#print "saved :%s" %mergedPickleName
newFSC.logger = temp
return newFSC
def collectStats( self, types, directory, fileType = "tx", startTime = '2006-05-18 00:00:00', endTime = "", interval = 60*StatsDateLib.MINUTE, save = True ):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger#Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None : #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile( self.pickleName ):
if self.logger != None :
self.logger.warning( _("User tried to modify allready filled pickle file." ) )
self.logger.warning( _("Pickle was named : %s") %self.pickleName )
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName( client = self.client, currentTime = startTime, machine = self.machine, fileType = fileType )
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats( endTime )
self.logger = temp
if save == True :# must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save ( object = self.statsCollection, filename = self.pickleName )
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname( self.pickleName )
while( dirname != STATSPATHS.STATSPICKLES[:-1] ):#[:-1] removes the last / character
try:
os.chmod( dirname, 0777 )
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced :
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info( _("Saved pickle named : %s ") %self.pickleName )
