def saveList( self, user, clients ): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir( directory ): os.makedirs( directory, mode=0777 ) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save( object = self.savedFileList, filename = completeFilename )
def saveList(self, user, clients): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir(directory): os.makedirs(directory, mode=0777) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save(object=self.savedFileList, filename=completeFilename)
def addAutomaticUpdateToLogs( self, timeOfUpdateInIsoFormat, currentUpdateFrequency = None ): """ @summary : Writes a new file in the log folder containing the current update frequency. @param timeOfUpdateInIsoFormat: Time that the entries name will sport. """ paths = StatsPaths() paths.setPaths() fileName = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + str( timeOfUpdateInIsoFormat ).replace( " ", "_" ) #Safety to make sure if not os.path.isdir( os.path.dirname( fileName ) ): os.makedirs( os.path.dirname( fileName ), 0777 ) if currentUpdateFrequency == None : currentUpdateFrequency = self.getCurrentUpdateFrequency() CpickleWrapper.save( currentUpdateFrequency, fileName ) allEntries = os.listdir(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/") allEntries.sort() entriesToRemove = allEntries[ :-self.numberOfLogsToKeep] for entrytoRemove in entriesToRemove: os.remove(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + entrytoRemove )
def saveAccessDictionary( self ): """ @summary: Saves the current accessDictionary into the accessfile. """ if not os.path.isdir( os.path.dirname( self.accessFile ) ): os.makedirs( os.path.dirname( self.accessFile ) ) CpickleWrapper.save( self.accessDictionary, self.accessFile )
def saveAccessDictionary(self): """ @summary: Saves the current accessDictionary into the accessfile. """ if not os.path.isdir(os.path.dirname(self.accessFile)): os.makedirs(os.path.dirname(self.accessFile)) CpickleWrapper.save(self.accessDictionary, self.accessFile)
def saveCurrentMachineParameters( machineParameters ): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir( os.path.dirname( paths.STATSPREVIOUSMACHINEPARAMS ) ): os.makedirs( os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS) ) CpickleWrapper.save( machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def saveCurrentMachineParameters(machineParameters): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)): os.makedirs(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)) CpickleWrapper.save(machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def loadAccessFile(self): """ @summary: Loads the accessFile into the accessDictionary. """ self.accessDictionary = CpickleWrapper.load(self.accessFile)
def getSavedList( self, user, clients ): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try : self.savedFileList = CpickleWrapper.load( directory + fileName ) if self.savedFileLis == None : self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def getSavedList(self, user, clients): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try: self.savedFileList = CpickleWrapper.load(directory + fileName) if self.savedFileLis == None: self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def loadAccessFile(self): """ @summary: Loads the accessFile into the accessDictionary. """ self.accessDictionary = CpickleWrapper.load( self.accessFile )
def printPickle(pickle, outputFile=""): """ Print content of a pickle file containing a FileStatscollector instance on the desired output. Default output is the console screen. File can be specified to make reading easier. """ if outputFile != "": fileHandle = open(outputFile, 'w') old_stdout = sys.stdout sys.stdout = fileHandle statsCollection = CpickleWrapper.load(pickle) print _("Pickle used : %s") % pickle print _("\n\nFiles used : %s") % statsCollection.files print _("Starting date: %s") % statsCollection.startTime print _("Interval: %s") % statsCollection.interval print _("End time : %s") % statsCollection.endTime print _("nbEntries : %s") % statsCollection.nbEntries for j in range(statsCollection.nbEntries): print _("\nEntry's interval : %s - %s ") % ( statsCollection.fileEntries[j].startTime, statsCollection.fileEntries[j].endTime) print _("Files : ") print statsCollection.fileEntries[j].files print _("Products : ") print statsCollection.fileEntries[j].values.productTypes print _("Values :") print statsCollection.fileEntries[j].values.dictionary print _("Means :") print statsCollection.fileEntries[j].means print _("Medians") print statsCollection.fileEntries[j].medians print _("Minimums") print statsCollection.fileEntries[j].minimums print _("Maximums") print statsCollection.fileEntries[j].maximums print _("Time where max occured :") print statsCollection.fileEntries[j].timesWhereMaxOccured print _("Total") print statsCollection.fileEntries[j].totals print _("Files over maximum latency") print statsCollection.fileEntries[j].filesOverMaxLatency if outputFile != "": fileHandle.close() sys.stdout = old_stdout #resets standard output
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\ endTime = "2006-07-31 19:00:00", client = "satnet",\ machine = "pdsPM", fileType = "tx" ): """ @summary : This method merges entire hourly pickles files together. @None : This does not support merging part of the data of pickles. """ if logger != None : logger.debug( _("Call to mergeHourlyPickles received.") ) logging = True else: logging = False pickles = [] entries = {} width = StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) startTime = StatsDateLib.getIsoWithRoundedHours( startTime ) seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=width, interval=60*StatsDateLib.MINUTE )[:-1]) for seperator in seperators : pickles.append( StatsPickler.buildThisHoursFileName( client = client, offset = 0, currentTime = seperator, machine = machine, fileType = fileType ) ) startingNumberOfEntries = 0 #print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) for pickle in pickles : if os.path.isfile( pickle ) : tempCollection = CpickleWrapper.load( pickle ) if tempCollection != None : for i in xrange( len( tempCollection.fileEntries ) ): entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i] startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries ) else: sys.exit() else: emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60, entries = {} ) for i in xrange( 60 ): entries[i + startingNumberOfEntries ] = emptyEntries [i] startingNumberOfEntries = startingNumberOfEntries + 60 #print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) statsCollection = FileStatsCollector( startTime = startTime , endTime = endTime, interval = StatsDateLib.MINUTE, totalWidth = width, fileEntries = entries,fileType= fileType, logger = logger, logging = logging ) return statsCollection
def printPickle( pickle, outputFile = "" ): """ Print content of a pickle file containing a FileStatscollector instance on the desired output. Default output is the console screen. File can be specified to make reading easier. """ if outputFile != "": fileHandle = open( outputFile , 'w' ) old_stdout = sys.stdout sys.stdout = fileHandle statsCollection = CpickleWrapper.load( pickle ) print _("Pickle used : %s" )%pickle print _("\n\nFiles used : %s" ) %statsCollection.files print _("Starting date: %s" ) % statsCollection.startTime print _("Interval: %s" ) %statsCollection.interval print _("End time : %s" ) %statsCollection.endTime print _("nbEntries : %s" ) %statsCollection.nbEntries for j in range( statsCollection.nbEntries ): print _("\nEntry's interval : %s - %s " ) %( statsCollection.fileEntries[j].startTime, statsCollection.fileEntries[j].endTime ) print _("Files : " ) print statsCollection.fileEntries[j].files print _("Products : " ) print statsCollection.fileEntries[j].values.productTypes print _("Values :" ) print statsCollection.fileEntries[j].values.dictionary print _("Means :" ) print statsCollection.fileEntries[j].means print _("Medians" ) print statsCollection.fileEntries[j].medians print _("Minimums" ) print statsCollection.fileEntries[j].minimums print _("Maximums" ) print statsCollection.fileEntries[j].maximums print _("Time where max occured :" ) print statsCollection.fileEntries[j].timesWhereMaxOccured print _("Total" ) print statsCollection.fileEntries[j].totals print _("Files over maximum latency" ) print statsCollection.fileEntries[j].filesOverMaxLatency if outputFile != "": fileHandle.close() sys.stdout = old_stdout #resets standard output
def changeInUpdateFrenquencyFoundDuringTimespan( self, startTime, endTime ): """ @summary : Searchs whether or not there was a change during the specified timespan. @param statTime : Start time in the iso format of the time span to survey. @param endTime : End time in the iso format of the time span to survey/ @return : True or false whether there was a change or not, plus the original frequency and the new frequency. """ changeWasMade = False paths = StatsPaths() paths.setPaths() updatesDoneDuringTimespan = self.__getAutomaticUpdatesDoneDuringTimeSpan( startTime, endTime ) updatesDoneDuringTimespan.sort() if updatesDoneDuringTimespan != []: fileName = paths.STATSTEMPAUTUPDTLOGS + str(updatesDoneDuringTimespan[0]).replace( " ", "_" ) originalUpdateFrequency = CpickleWrapper.load(fileName) newUpdateFrequency = originalUpdateFrequency for update in updatesDoneDuringTimespan: fileName = paths.STATSTEMPAUTUPDTLOGS + str(update).replace( " ", "_" ) newUpdateFrequency = CpickleWrapper.load(fileName) if newUpdateFrequency != originalUpdateFrequency: changeWasMade = True break return changeWasMade, originalUpdateFrequency, newUpdateFrequency
def getMachineParametersFromPreviousCall() : """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile( paths.STATSPREVIOUSMACHINEPARAMS ): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS ) return previousMachineParams
def getMachineParametersFromPreviousCall(): """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile(paths.STATSPREVIOUSMACHINEPARAMS): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS) return previousMachineParams
def previousUpdateFrequency(self): """ @summary : Finds and returns the frequency of the previous update. @return : The freqency of the previous update. """ paths = StatsPaths() paths.setPaths() lastUpdate = self.getTimeOfLastUpdateInLogs() fileName = paths.STATSTEMPAUTUPDTLOGS + str(lastUpdate).replace( " ", "_" ) lastUpdateFrequency = CpickleWrapper.load(fileName) return lastUpdateFrequency
def main(): """ Test cases. These test must work. Please run the tests after modifying this file to make sure tests still work. """ import commands ################################################################################################## # # This section test the utility methods which do not require log files # ################################################################################################## updateManager = AutomaticUpdatesManager( 10, "pxStatsStartup" ) print "" print "" print "getMissingDaysBetweenUpdates test #1 : " print "" print """updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] ") print "Obtained result : %s " %updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) if not updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) ==\ ['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.") print "" print "" print "getMissingWeeksBetweenUpdates test #1 : " print "" print """updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00']") print "Obtained result : %s " %updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) if not updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) ==\ ['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.") print "" print "" print "getMissingMonthssBetweenUpdates test #1 : " print "" print """updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00','2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ") print "Obtained result : %s " %updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) if not (updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00', '2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ) : raise AssertionError("getMissingMonthsBetweenUpdates test #1 is broken.") print "" print "" print "getMissingYearsBetweenUpdates test #1 : " print "" print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']") print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.") print "" print "" print "getMissingYearsBetweenUpdates test #1 : " print "" print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']") print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.") print "" print "" print "getCurrentUpdateFrequency() test #1" print "" print """ updateManager.getCurrentUpdateFrequency() """ print "PLEASE VERIFY THAT RESULT MATCH UPDATE FREQUENCY FOUND IN CRONTAB ENTRY." print "Type crontab -l to see current crontab configuration." print "Obtained result : %s" %( updateManager.getCurrentUpdateFrequency() ) print "" ################################################################################################## # # This section requires log files. If log files currently exist, will temporarily copy # them to a temp folder. Otherwise, will create some test updates files usefull only for # testing. # ################################################################################################## paths = StatsPaths( ) paths.setPaths() print paths.STATSTEMPAUTUPDTLOGS if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS ) : print commands.getoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS, paths.STATSTEMPAUTUPDTLOGS[:-1] + ".old" ) ) os.makedirs( paths.STATSTEMPAUTUPDTLOGS, 0777 ) print "" print "" print "getTimeOfLastUpdateInLogs test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.getTimeOfLastUpdateInLogs() """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) updateManager.getTimeOfLastUpdateInLogs() print "Expected results : 2008-02-01 04:00:00" print "Obtained results : %s" %updateManager.getTimeOfLastUpdateInLogs( ) if not updateManager.getTimeOfLastUpdateInLogs( ) == "2008-02-01 04:00:00" : raise AssertionError( "getTimeOfLastUpdateInLogs test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "isFirstUpdateOfTheDay test#1( valid case )" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" ) : Create an update for that day """ print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day""" updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" )) if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-14_23:00:00" ) print "" print "" print "isFirstUpdateOfTheDay test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" ) : Create an update for that day """ print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day""" updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" )) if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-15_23:00:00" ) print "" print "" print "isFirstUpdateOfTheWeek test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" ) : Create an update for that week """ print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week""" updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheWeek test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that week """ print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week""" updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "isFirstUpdateOfTheMonth test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" ) : Create an update for that Month """ print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month""" updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) ) if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-01-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheMonth test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Month """ print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month""" updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "isFirstUpdateOfTheYear test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" ) : Create an update for that Year """ print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year""" updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheYear test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-01-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheYear test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Year """ print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year""" updateManager.addAutomaticUpdateToLogs( "2008-02-27_23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "getTimeSinceLastUpdate test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update to test with""" print """updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) : Date to test with """ updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : " print "Obtained result : %s" %( updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) ) if not updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) == 86400 : raise AssertionError( "getTimeSinceLastUpdate test#1 is broken" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "getNbAutomaticUpdatesDoneDuringTimeSpan test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-01-01 00:50:00","2008-02-01 03:30:00") """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) print "Expected results : 3" print "Obtained results : %s" %updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00") if not updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00") == 3 : raise AssertionError( "getNbAutomaticUpdatesDoneDuringTimeSpan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "changeInUpdateFrenquencyFoundDuringTimespan test#1(no change to be detected)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")""" updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) print "Expected results : False" print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == (False, updateManager.getCurrentUpdateFrequency(), updateManager.getCurrentUpdateFrequency()) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "changeInUpdateFrenquencyFoundDuringTimespan test#2(Frequency change to be detected)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """CpickleWrapper.save(0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00") : Corrupt an entry for testing purposes""" print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")""" updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) CpickleWrapper.save(-1, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00") print "Expected results : True %s -1" %updateManager.getCurrentUpdateFrequency() print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-05 05:00:00") if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == ( True, updateManager.getCurrentUpdateFrequency(), -1 ) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "previousUpdateFrequency test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.previousUpdateFrequency() """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) CpickleWrapper.save( 0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) print "Expected results : 0 " print "Obtained results : %s" %updateManager.previousUpdateFrequency() if not updateManager.previousUpdateFrequency() == 0 : raise AssertionError( "previousUpdateFrequency test#1 is broken" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.removedirs( paths.STATSTEMPAUTUPDTLOGS ) if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS + ".old" ) : commands.getstatusoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS + ".old", paths.STATSTEMPAUTUPDTLOGS ) )
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None: logger.debug(_("Call to mergePickles received.")) logging = True else: logging = False entryList = [] for pickle in pickleNames: #for every pickle we eneed to merge if os.path.isfile(pickle): entryList.append(CpickleWrapper.load(pickle)) else: #Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentTime) + StatsDateLib.HOUR) entryList.append( FileStatsCollector(startTime=currentTime, endTime=endTime, logger=logger, logging=logging)) if logger != None: logger.warning( _("Pickle named %s did not exist. Empty entry was used instead." ) % pickle) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid(entryList) == True: for i in range(1, len(entryList)): #add other entries for file in entryList[i].files: if file not in newFSC.files: newFSC.files.append(file) for j in range(len(newFSC.fileEntries)): # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[ j].nbFiles + (newFSC.fileEntries[j].nbFiles) for type in newFSC.statsTypes: newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[ j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket=0, finishingBucket=newFSC.nbEntries - 1) else: #Did not merge pickles named. Pickle list was not valid." if logger != None: logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid." ) % pickleNames) logger.warning( _("Filled with empty entries instead.") % pickleNames) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries=60, entries={}) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save(newFSC, mergedPickleName) try: os.chmod(mergedPickleName, 0777) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\ endTime = "2006-07-31 19:00:00", client = "satnet",\ machine = "pdsPM", fileType = "tx" ): """ @summary : This method merges entire hourly pickles files together. @None : This does not support merging part of the data of pickles. """ if logger != None: logger.debug(_("Call to mergeHourlyPickles received.")) logging = True else: logging = False pickles = [] entries = {} width = StatsDateLib.getSecondsSinceEpoch( endTime) - StatsDateLib.getSecondsSinceEpoch(startTime) startTime = StatsDateLib.getIsoWithRoundedHours(startTime) seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime(startTime=startTime, width=width, interval=60 * StatsDateLib.MINUTE)[:-1]) for seperator in seperators: pickles.append( StatsPickler.buildThisHoursFileName(client=client, offset=0, currentTime=seperator, machine=machine, fileType=fileType)) startingNumberOfEntries = 0 #print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) for pickle in pickles: if os.path.isfile(pickle): tempCollection = CpickleWrapper.load(pickle) if tempCollection != None: for i in xrange(len(tempCollection.fileEntries)): entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i] startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries) else: sys.exit() else: emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries=60, entries={}) for i in xrange(60): entries[i + startingNumberOfEntries] = emptyEntries[i] startingNumberOfEntries = startingNumberOfEntries + 60 #print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) statsCollection = FileStatsCollector(startTime=startTime, endTime=endTime, interval=StatsDateLib.MINUTE, totalWidth=width, fileEntries=entries, fileType=fileType, logger=logger, logging=logging) return statsCollection
""" small test case. Tests if everything works plus gives an idea on proper usage. """ import time statsPaths = StatsPaths() statsPaths.setPaths() timeA= time.time() types = [ "latency", "errors","bytecount" ] filename = statsPaths.STATSLOGS + 'someMachine/tx_someclient.log' startingHours=["00:00:00","01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00" ] endingHours = ["01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00", "00:00:00" ] stats = FileStatsCollector( files = [ filename ], statsTypes = types , startTime = '2007-06-12 %s' %startingHours[2], endTime = '2007-06-12 %s' %endingHours[10], interval = 1*MINUTE, totalWidth = 9*HOUR ) stats.collectStats() timeB = time.time() print timeB - timeA saveFile = STATSPATHS.STATSDATA + "testNew" del stats.logger CpickleWrapper.save( object = stats, filename = saveFile )
def collectStats(self, types, directory, fileType="tx", startTime='2006-05-18 00:00:00', endTime="", interval=60 * StatsDateLib.MINUTE, save=True): """ @summary : This method is used to collect stats from logfiles found within a directory. Types is the type of dats to be collected. Pickle is the name of the file to be used. If not specified will be generated according to the other parameters. FileType specifies what type of files to look for in the directory. StartTime and endtime specify the boundaries within wich we'll collect the data. Interval the width of the entries in the stats collection save can be false if for some reason user does not want to save pickle. If both of the above are true, hourly pickles will be done. Pre-conditions : StarTime needs to be smaller than endTime. If Daily pickling is used,width between start and endTime needs to be no more than 24hours If Hourly pickling is used,width between start and endTime needs to be no more than 1hour. If pre-conditions aren't met, application will fail. """ global _ #Find up to date file list. self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\ logType = fileType, name = self.client, logger = self.logger, logging = self.logging ) temp = self.logger #Need to remove current logger temporarily del self.logger self.fileCollection.collectEntries() #find all entries from the folder self.logger = temp if self.fileCollection.logger != None: #No longer need the logger self.fileCollection.logger = None if os.path.isfile(self.pickleName): if self.logger != None: self.logger.warning( _("User tried to modify allready filled pickle file.")) self.logger.warning( _("Pickle was named : %s") % self.pickleName) # Creates a new FileStats collector wich spans from the very # start of the hour up till the end. if self.pickleName == "": self.pickleName = StatsPickler.buildThisHoursFileName( client=self.client, currentTime=startTime, machine=self.machine, fileType=fileType) self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\ startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\ interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging ) #Temporarily delete logger to make sure no duplicated lines appears in log file. temp = self.logger del self.logger self.statsCollection.collectStats(endTime) self.logger = temp if save == True: # must remove logger temporarily. Cannot saved opened files. if self.statsCollection.logger != None: temp = self.statsCollection.logger del self.statsCollection.logger loggerNeedsToBeReplaced = True CpickleWrapper.save(object=self.statsCollection, filename=self.pickleName) try: os.chmod(self.pickleName, 0777) dirname = os.path.dirname(self.pickleName) while (dirname != STATSPATHS.STATSPICKLES[:-1] ): #[:-1] removes the last / character try: os.chmod(dirname, 0777) except: pass dirname = os.path.dirname(dirname) except: pass if loggerNeedsToBeReplaced: self.statsCollection.logger = temp if self.logger != None: self.logger.info( _("Saved pickle named : %s ") % self.pickleName)
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None : logger.debug( _("Call to mergePickles received.") ) logging = True else: logging = False entryList = [] for pickle in pickleNames:#for every pickle we eneed to merge if os.path.isfile( pickle ): entryList.append( CpickleWrapper.load( pickle ) ) else:#Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( currentTime ) + StatsDateLib.HOUR ) entryList.append( FileStatsCollector( startTime = currentTime, endTime = endTime,logger =logger, logging =logging ) ) if logger != None : logger.warning( _("Pickle named %s did not exist. Empty entry was used instead.") %pickle ) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid( entryList ) == True : for i in range ( 1 , len( entryList ) ): #add other entries for file in entryList[i].files : if file not in newFSC.files : newFSC.files.append( file ) for j in range( len( newFSC.fileEntries ) ) : # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes ) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files ) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times ) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[j].nbFiles + ( newFSC.fileEntries[ j ].nbFiles) for type in newFSC.statsTypes : newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket = 0 , finishingBucket = newFSC.nbEntries -1 ) else:#Did not merge pickles named. Pickle list was not valid." if logger != None : logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid.") %pickleNames ) logger.warning( _("Filled with empty entries instead.") %pickleNames ) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60 , entries = {} ) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save( newFSC, mergedPickleName ) try: os.chmod( mergedPickleName, 0777 ) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def collectStats( self, types, directory, fileType = "tx", startTime = '2006-05-18 00:00:00', endTime = "", interval = 60*StatsDateLib.MINUTE, save = True ): """ @summary : This method is used to collect stats from logfiles found within a directory. Types is the type of dats to be collected. Pickle is the name of the file to be used. If not specified will be generated according to the other parameters. FileType specifies what type of files to look for in the directory. StartTime and endtime specify the boundaries within wich we'll collect the data. Interval the width of the entries in the stats collection save can be false if for some reason user does not want to save pickle. If both of the above are true, hourly pickles will be done. Pre-conditions : StarTime needs to be smaller than endTime. If Daily pickling is used,width between start and endTime needs to be no more than 24hours If Hourly pickling is used,width between start and endTime needs to be no more than 1hour. If pre-conditions aren't met, application will fail. """ global _ #Find up to date file list. self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\ logType = fileType, name = self.client, logger = self.logger, logging = self.logging ) temp = self.logger#Need to remove current logger temporarily del self.logger self.fileCollection.collectEntries() #find all entries from the folder self.logger = temp if self.fileCollection.logger != None : #No longer need the logger self.fileCollection.logger = None if os.path.isfile( self.pickleName ): if self.logger != None : self.logger.warning( _("User tried to modify allready filled pickle file." ) ) self.logger.warning( _("Pickle was named : %s") %self.pickleName ) # Creates a new FileStats collector wich spans from the very # start of the hour up till the end. if self.pickleName == "": self.pickleName = StatsPickler.buildThisHoursFileName( client = self.client, currentTime = startTime, machine = self.machine, fileType = fileType ) self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\ startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\ interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging ) #Temporarily delete logger to make sure no duplicated lines appears in log file. temp = self.logger del self.logger self.statsCollection.collectStats( endTime ) self.logger = temp if save == True :# must remove logger temporarily. Cannot saved opened files. if self.statsCollection.logger != None: temp = self.statsCollection.logger del self.statsCollection.logger loggerNeedsToBeReplaced = True CpickleWrapper.save ( object = self.statsCollection, filename = self.pickleName ) try: os.chmod(self.pickleName, 0777) dirname = os.path.dirname( self.pickleName ) while( dirname != STATSPATHS.STATSPICKLES[:-1] ):#[:-1] removes the last / character try: os.chmod( dirname, 0777 ) except: pass dirname = os.path.dirname(dirname) except: pass if loggerNeedsToBeReplaced : self.statsCollection.logger = temp if self.logger != None: self.logger.info( _("Saved pickle named : %s ") %self.pickleName )