def saveList( self, user, clients ): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir( directory ): os.makedirs( directory, mode=0777 ) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save( object = self.savedFileList, filename = completeFilename )
def addAutomaticUpdateToLogs( self, timeOfUpdateInIsoFormat, currentUpdateFrequency = None ): """ @summary : Writes a new file in the log folder containing the current update frequency. @param timeOfUpdateInIsoFormat: Time that the entries name will sport. """ paths = StatsPaths() paths.setPaths() fileName = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + str( timeOfUpdateInIsoFormat ).replace( " ", "_" ) #Safety to make sure if not os.path.isdir( os.path.dirname( fileName ) ): os.makedirs( os.path.dirname( fileName ), 0777 ) if currentUpdateFrequency == None : currentUpdateFrequency = self.getCurrentUpdateFrequency() CpickleWrapper.save( currentUpdateFrequency, fileName ) allEntries = os.listdir(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/") allEntries.sort() entriesToRemove = allEntries[ :-self.numberOfLogsToKeep] for entrytoRemove in entriesToRemove: os.remove(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + entrytoRemove )
def saveList(self, user, clients): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir(directory): os.makedirs(directory, mode=0777) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save(object=self.savedFileList, filename=completeFilename)
def saveAccessDictionary( self ): """ @summary: Saves the current accessDictionary into the accessfile. """ if not os.path.isdir( os.path.dirname( self.accessFile ) ): os.makedirs( os.path.dirname( self.accessFile ) ) CpickleWrapper.save( self.accessDictionary, self.accessFile )
def saveAccessDictionary(self): """ @summary: Saves the current accessDictionary into the accessfile. """ if not os.path.isdir(os.path.dirname(self.accessFile)): os.makedirs(os.path.dirname(self.accessFile)) CpickleWrapper.save(self.accessDictionary, self.accessFile)
def saveCurrentMachineParameters( machineParameters ): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir( os.path.dirname( paths.STATSPREVIOUSMACHINEPARAMS ) ): os.makedirs( os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS) ) CpickleWrapper.save( machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def saveCurrentMachineParameters(machineParameters): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)): os.makedirs(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)) CpickleWrapper.save(machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None : logger.debug( _("Call to mergePickles received.") ) logging = True else: logging = False entryList = [] for pickle in pickleNames:#for every pickle we eneed to merge if os.path.isfile( pickle ): entryList.append( CpickleWrapper.load( pickle ) ) else:#Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( currentTime ) + StatsDateLib.HOUR ) entryList.append( FileStatsCollector( startTime = currentTime, endTime = endTime,logger =logger, logging =logging ) ) if logger != None : logger.warning( _("Pickle named %s did not exist. Empty entry was used instead.") %pickle ) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid( entryList ) == True : for i in range ( 1 , len( entryList ) ): #add other entries for file in entryList[i].files : if file not in newFSC.files : newFSC.files.append( file ) for j in range( len( newFSC.fileEntries ) ) : # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes ) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files ) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times ) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[j].nbFiles + ( newFSC.fileEntries[ j ].nbFiles) for type in newFSC.statsTypes : newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket = 0 , finishingBucket = newFSC.nbEntries -1 ) else:#Did not merge pickles named. Pickle list was not valid." if logger != None : logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid.") %pickleNames ) logger.warning( _("Filled with empty entries instead.") %pickleNames ) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60 , entries = {} ) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save( newFSC, mergedPickleName ) try: os.chmod( mergedPickleName, 0777 ) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None: logger.debug(_("Call to mergePickles received.")) logging = True else: logging = False entryList = [] for pickle in pickleNames: #for every pickle we eneed to merge if os.path.isfile(pickle): entryList.append(CpickleWrapper.load(pickle)) else: #Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentTime) + StatsDateLib.HOUR) entryList.append( FileStatsCollector(startTime=currentTime, endTime=endTime, logger=logger, logging=logging)) if logger != None: logger.warning( _("Pickle named %s did not exist. Empty entry was used instead." ) % pickle) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid(entryList) == True: for i in range(1, len(entryList)): #add other entries for file in entryList[i].files: if file not in newFSC.files: newFSC.files.append(file) for j in range(len(newFSC.fileEntries)): # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[ j].nbFiles + (newFSC.fileEntries[j].nbFiles) for type in newFSC.statsTypes: newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[ j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket=0, finishingBucket=newFSC.nbEntries - 1) else: #Did not merge pickles named. Pickle list was not valid." if logger != None: logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid." ) % pickleNames) logger.warning( _("Filled with empty entries instead.") % pickleNames) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries=60, entries={}) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save(newFSC, mergedPickleName) try: os.chmod(mergedPickleName, 0777) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def main(): """ Test cases. These test must work. Please run the tests after modifying this file to make sure tests still work. """ import commands ################################################################################################## # # This section test the utility methods which do not require log files # ################################################################################################## updateManager = AutomaticUpdatesManager( 10, "pxStatsStartup" ) print "" print "" print "getMissingDaysBetweenUpdates test #1 : " print "" print """updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] ") print "Obtained result : %s " %updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) if not updateManager.getMissingDaysBetweenUpdates( "2008-02-10 15:00:00", "2008-02-15 23:00:00" ) ==\ ['2008-02-10 15:00:00', '2008-02-11 15:00:00', '2008-02-12 15:00:00', '2008-02-13 15:00:00', '2008-02-14 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.") print "" print "" print "getMissingWeeksBetweenUpdates test #1 : " print "" print """updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00']") print "Obtained result : %s " %updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) if not updateManager.getMissingWeeksBetweenUpdates( "2008-02-10 15:00:00", "2008-03-15 23:00:00" ) ==\ ['2008-02-10 15:00:00', '2008-02-17 15:00:00', '2008-02-24 15:00:00', '2008-03-02 15:00:00'] : raise AssertionError("getIsoFromEpoch test #1 is broken.") print "" print "" print "getMissingMonthssBetweenUpdates test #1 : " print "" print """updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00','2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ") print "Obtained result : %s " %updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) if not (updateManager.getMissingMonthsBetweenUpdates( "2008-02-10 15:00:00", "2009-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2008-03-10 15:00:00', '2008-04-10 15:00:00', '2008-05-10 15:00:00', '2008-06-10 15:00:00', '2008-07-10 15:00:00', '2008-08-10 15:00:00', '2008-09-10 15:00:00', '2008-10-10 15:00:00', '2008-11-10 15:00:00', '2008-12-10 15:00:00', '2009-01-10 15:00:00', '2009-02-10 15:00:00', '2009-03-10 15:00:00', '2009-04-10 15:00:00', '2009-05-10 15:00:00'] ) : raise AssertionError("getMissingMonthsBetweenUpdates test #1 is broken.") print "" print "" print "getMissingYearsBetweenUpdates test #1 : " print "" print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']") print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.") print "" print "" print "getMissingYearsBetweenUpdates test #1 : " print "" print """updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) : """ print "Expected result : %s " %("['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00']") print "Obtained result : %s " %updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) if not updateManager.getMissingYearsBetweenUpdates( "2008-02-10 15:00:00", "2011-06-15 23:00:00" ) == ['2008-02-10 15:00:00', '2009-02-10 15:00:00', '2010-02-10 15:00:00'] : raise AssertionError("getMissingYearsBetweenUpdates test #1.") print "" print "" print "getCurrentUpdateFrequency() test #1" print "" print """ updateManager.getCurrentUpdateFrequency() """ print "PLEASE VERIFY THAT RESULT MATCH UPDATE FREQUENCY FOUND IN CRONTAB ENTRY." print "Type crontab -l to see current crontab configuration." print "Obtained result : %s" %( updateManager.getCurrentUpdateFrequency() ) print "" ################################################################################################## # # This section requires log files. If log files currently exist, will temporarily copy # them to a temp folder. Otherwise, will create some test updates files usefull only for # testing. # ################################################################################################## paths = StatsPaths( ) paths.setPaths() print paths.STATSTEMPAUTUPDTLOGS if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS ) : print commands.getoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS, paths.STATSTEMPAUTUPDTLOGS[:-1] + ".old" ) ) os.makedirs( paths.STATSTEMPAUTUPDTLOGS, 0777 ) print "" print "" print "getTimeOfLastUpdateInLogs test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.getTimeOfLastUpdateInLogs() """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) updateManager.getTimeOfLastUpdateInLogs() print "Expected results : 2008-02-01 04:00:00" print "Obtained results : %s" %updateManager.getTimeOfLastUpdateInLogs( ) if not updateManager.getTimeOfLastUpdateInLogs( ) == "2008-02-01 04:00:00" : raise AssertionError( "getTimeOfLastUpdateInLogs test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "isFirstUpdateOfTheDay test#1( valid case )" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" ) : Create an update for that day """ print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day""" updateManager.addAutomaticUpdateToLogs( "2007-12-14 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" )) if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-14_23:00:00" ) print "" print "" print "isFirstUpdateOfTheDay test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" ) : Create an update for that day """ print """updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) : With only that update, test if it's first of the day""" updateManager.addAutomaticUpdateToLogs( "2007-12-15 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" )) if not updateManager.isFirstUpdateOfTheDay( "2007-12-15 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheDay test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-12-15_23:00:00" ) print "" print "" print "isFirstUpdateOfTheWeek test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" ) : Create an update for that week """ print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week""" updateManager.addAutomaticUpdateToLogs( "2008-02-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheWeek test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that week """ print """updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the week""" updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheWeek test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "isFirstUpdateOfTheMonth test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" ) : Create an update for that Month """ print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month""" updateManager.addAutomaticUpdateToLogs( "2008-01-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) ) if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-01-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheMonth test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Month """ print """updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Month""" updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheWeek( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheMonth( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "isFirstUpdateOfTheYear test#1(valid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" ) : Create an update for that Year """ print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year""" updateManager.addAutomaticUpdateToLogs( "2007-01-20 23:00:00" ) print "Expected result : True" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == True : raise AssertionError("isFirstUpdateOfTheYear test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2007-01-20_23:00:00" ) print "" print "" print "isFirstUpdateOfTheYear test#2(invalid case)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update for that Year """ print """updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) : With only that update, test if it's first of the Year""" updateManager.addAutomaticUpdateToLogs( "2008-02-27_23:00:00" ) print "Expected result : False" print "Obtained result : %s" %(updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" )) if not updateManager.isFirstUpdateOfTheYear( "2008-02-27 23:00:00" ) == False : raise AssertionError("isFirstUpdateOfTheMonth test#1 is broken") os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "getTimeSinceLastUpdate test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) : Create an update to test with""" print """updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) : Date to test with """ updateManager.addAutomaticUpdateToLogs( "2008-02-27 23:00:00" ) print "Expected result : " print "Obtained result : %s" %( updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) ) if not updateManager.getTimeSinceLastUpdate( "2008-02-28 23:00:00" ) == 86400 : raise AssertionError( "getTimeSinceLastUpdate test#1 is broken" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-27_23:00:00" ) print "" print "" print "getNbAutomaticUpdatesDoneDuringTimeSpan test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-01-01 00:50:00","2008-02-01 03:30:00") """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) print "Expected results : 3" print "Obtained results : %s" %updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00") if not updateManager.getNbAutomaticUpdatesDoneDuringTimeSpan("2008-02-01 00:50:00","2008-02-01 03:30:00") == 3 : raise AssertionError( "getNbAutomaticUpdatesDoneDuringTimeSpan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "changeInUpdateFrenquencyFoundDuringTimespan test#1(no change to be detected)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")""" updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) print "Expected results : False" print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == (False, updateManager.getCurrentUpdateFrequency(), updateManager.getCurrentUpdateFrequency()) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "changeInUpdateFrenquencyFoundDuringTimespan test#2(Frequency change to be detected)" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) : Create a series of update logs""" print """CpickleWrapper.save(0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00") : Corrupt an entry for testing purposes""" print """updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00")""" updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 03:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 04:00:00" ) CpickleWrapper.save(-1, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00") print "Expected results : True %s -1" %updateManager.getCurrentUpdateFrequency() print "Obtained results : %s %s %s" % updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-05 05:00:00") if not updateManager.changeInUpdateFrenquencyFoundDuringTimespan("2008-02-01 00:00:00", "2008-02-01 05:00:00") == ( True, updateManager.getCurrentUpdateFrequency(), -1 ) : raise AssertionError( "changeInUpdateFrenquencyFoundDuringTimespan test#1 is broken" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_03:00:00" ) os.remove(paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_04:00:00" ) print "" print "" print "previousUpdateFrequency test#1" print "" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) : Create a series of update logs""" print """updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) : Create a series of update logs""" print """updateManager.previousUpdateFrequency() """ updateManager.addAutomaticUpdateToLogs( "2008-02-01 00:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 01:00:00" ) updateManager.addAutomaticUpdateToLogs( "2008-02-01 02:00:00" ) CpickleWrapper.save( 0, paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) print "Expected results : 0 " print "Obtained results : %s" %updateManager.previousUpdateFrequency() if not updateManager.previousUpdateFrequency() == 0 : raise AssertionError( "previousUpdateFrequency test#1 is broken" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_00:00:00" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_01:00:00" ) os.remove( paths.STATSTEMPAUTUPDTLOGS + "2008-02-01_02:00:00" ) os.removedirs( paths.STATSTEMPAUTUPDTLOGS ) if os.path.isdir( paths.STATSTEMPAUTUPDTLOGS + ".old" ) : commands.getstatusoutput( "mv %s %s" %( paths.STATSTEMPAUTUPDTLOGS + ".old", paths.STATSTEMPAUTUPDTLOGS ) )
def collectStats(self, types, directory, fileType="tx", startTime='2006-05-18 00:00:00', endTime="", interval=60 * StatsDateLib.MINUTE, save=True): """ @summary : This method is used to collect stats from logfiles found within a directory. Types is the type of dats to be collected. Pickle is the name of the file to be used. If not specified will be generated according to the other parameters. FileType specifies what type of files to look for in the directory. StartTime and endtime specify the boundaries within wich we'll collect the data. Interval the width of the entries in the stats collection save can be false if for some reason user does not want to save pickle. If both of the above are true, hourly pickles will be done. Pre-conditions : StarTime needs to be smaller than endTime. If Daily pickling is used,width between start and endTime needs to be no more than 24hours If Hourly pickling is used,width between start and endTime needs to be no more than 1hour. If pre-conditions aren't met, application will fail. """ global _ #Find up to date file list. self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\ logType = fileType, name = self.client, logger = self.logger, logging = self.logging ) temp = self.logger #Need to remove current logger temporarily del self.logger self.fileCollection.collectEntries() #find all entries from the folder self.logger = temp if self.fileCollection.logger != None: #No longer need the logger self.fileCollection.logger = None if os.path.isfile(self.pickleName): if self.logger != None: self.logger.warning( _("User tried to modify allready filled pickle file.")) self.logger.warning( _("Pickle was named : %s") % self.pickleName) # Creates a new FileStats collector wich spans from the very # start of the hour up till the end. if self.pickleName == "": self.pickleName = StatsPickler.buildThisHoursFileName( client=self.client, currentTime=startTime, machine=self.machine, fileType=fileType) self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\ startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\ interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging ) #Temporarily delete logger to make sure no duplicated lines appears in log file. temp = self.logger del self.logger self.statsCollection.collectStats(endTime) self.logger = temp if save == True: # must remove logger temporarily. Cannot saved opened files. if self.statsCollection.logger != None: temp = self.statsCollection.logger del self.statsCollection.logger loggerNeedsToBeReplaced = True CpickleWrapper.save(object=self.statsCollection, filename=self.pickleName) try: os.chmod(self.pickleName, 0777) dirname = os.path.dirname(self.pickleName) while (dirname != STATSPATHS.STATSPICKLES[:-1] ): #[:-1] removes the last / character try: os.chmod(dirname, 0777) except: pass dirname = os.path.dirname(dirname) except: pass if loggerNeedsToBeReplaced: self.statsCollection.logger = temp if self.logger != None: self.logger.info( _("Saved pickle named : %s ") % self.pickleName)
""" small test case. Tests if everything works plus gives an idea on proper usage. """ import time statsPaths = StatsPaths() statsPaths.setPaths() timeA= time.time() types = [ "latency", "errors","bytecount" ] filename = statsPaths.STATSLOGS + 'someMachine/tx_someclient.log' startingHours=["00:00:00","01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00" ] endingHours = ["01:00:00","02:00:00","03:00:00","04:00:00","05:00:00","06:00:00","07:00:00","08:00:00","09:00:00","10:00:00","11:00:00","12:00:00","13:00:00","14:00:00","15:00:00","16:00:00","17:00:00","18:00:00","19:00:00","20:00:00","21:00:00","22:00:00","23:00:00", "00:00:00" ] stats = FileStatsCollector( files = [ filename ], statsTypes = types , startTime = '2007-06-12 %s' %startingHours[2], endTime = '2007-06-12 %s' %endingHours[10], interval = 1*MINUTE, totalWidth = 9*HOUR ) stats.collectStats() timeB = time.time() print timeB - timeA saveFile = STATSPATHS.STATSDATA + "testNew" del stats.logger CpickleWrapper.save( object = stats, filename = saveFile )
def collectStats( self, types, directory, fileType = "tx", startTime = '2006-05-18 00:00:00', endTime = "", interval = 60*StatsDateLib.MINUTE, save = True ): """ @summary : This method is used to collect stats from logfiles found within a directory. Types is the type of dats to be collected. Pickle is the name of the file to be used. If not specified will be generated according to the other parameters. FileType specifies what type of files to look for in the directory. StartTime and endtime specify the boundaries within wich we'll collect the data. Interval the width of the entries in the stats collection save can be false if for some reason user does not want to save pickle. If both of the above are true, hourly pickles will be done. Pre-conditions : StarTime needs to be smaller than endTime. If Daily pickling is used,width between start and endTime needs to be no more than 24hours If Hourly pickling is used,width between start and endTime needs to be no more than 1hour. If pre-conditions aren't met, application will fail. """ global _ #Find up to date file list. self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\ logType = fileType, name = self.client, logger = self.logger, logging = self.logging ) temp = self.logger#Need to remove current logger temporarily del self.logger self.fileCollection.collectEntries() #find all entries from the folder self.logger = temp if self.fileCollection.logger != None : #No longer need the logger self.fileCollection.logger = None if os.path.isfile( self.pickleName ): if self.logger != None : self.logger.warning( _("User tried to modify allready filled pickle file." ) ) self.logger.warning( _("Pickle was named : %s") %self.pickleName ) # Creates a new FileStats collector wich spans from the very # start of the hour up till the end. if self.pickleName == "": self.pickleName = StatsPickler.buildThisHoursFileName( client = self.client, currentTime = startTime, machine = self.machine, fileType = fileType ) self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\ startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\ interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging ) #Temporarily delete logger to make sure no duplicated lines appears in log file. temp = self.logger del self.logger self.statsCollection.collectStats( endTime ) self.logger = temp if save == True :# must remove logger temporarily. Cannot saved opened files. if self.statsCollection.logger != None: temp = self.statsCollection.logger del self.statsCollection.logger loggerNeedsToBeReplaced = True CpickleWrapper.save ( object = self.statsCollection, filename = self.pickleName ) try: os.chmod(self.pickleName, 0777) dirname = os.path.dirname( self.pickleName ) while( dirname != STATSPATHS.STATSPICKLES[:-1] ):#[:-1] removes the last / character try: os.chmod( dirname, 0777 ) except: pass dirname = os.path.dirname(dirname) except: pass if loggerNeedsToBeReplaced : self.statsCollection.logger = temp if self.logger != None: self.logger.info( _("Saved pickle named : %s ") %self.pickleName )