def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\ pickleName = "", logger = None, logging = False, machine = "pdsCSP" ): """ Constructor. -Builds a StatsPickler with no entries. """ self.client = client # Name of the client for whom were collecting data self.pickleName = "" # Pickle self.directory = directory # Name of the directory containing stats files. self.statsTypes = statsTypes or [] # Types we'll search for stats. self.machine = machine # Machine on wich the data resides. self.loggerName = 'pickling' # Name of the logger. self.logger = logger # Permits a logging system for this object. self.logging = logging # Whether or not to enable logging. if self.logging == True: if logger is None: # Enable logging if not os.path.isdir(STATSPATHS.STATSLOGGING): os.makedirs(STATSPATHS.STATSLOGGING, mode=0777) self.logger = Logger(STATSPATHS.STATSLOGGING + 'stats_' + self.loggerName + '.log.notb', 'INFO', 'TX' + self.loggerName, bytes=True) self.logger = self.logger.getLogger() else: logger = True self.statsCollection = statsCollection or FileStatsCollector( logger=self.logger, logging=logging) self.fileCollection = LogFileCollector(directory=directory, logger=self.logger, logging=logging) global _ _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
def collectStats(self, types, directory, fileType="tx", startTime='2006-05-18 00:00:00', endTime="", interval=60 * StatsDateLib.MINUTE, save=True): """ @summary : This method is used to collect stats from logfiles found within a directory. Types is the type of dats to be collected. Pickle is the name of the file to be used. If not specified will be generated according to the other parameters. FileType specifies what type of files to look for in the directory. StartTime and endtime specify the boundaries within wich we'll collect the data. Interval the width of the entries in the stats collection save can be false if for some reason user does not want to save pickle. If both of the above are true, hourly pickles will be done. Pre-conditions : StarTime needs to be smaller than endTime. If Daily pickling is used,width between start and endTime needs to be no more than 24hours If Hourly pickling is used,width between start and endTime needs to be no more than 1hour. If pre-conditions aren't met, application will fail. """ global _ #Find up to date file list. self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\ logType = fileType, name = self.client, logger = self.logger, logging = self.logging ) temp = self.logger #Need to remove current logger temporarily del self.logger self.fileCollection.collectEntries() #find all entries from the folder self.logger = temp if self.fileCollection.logger != None: #No longer need the logger self.fileCollection.logger = None if os.path.isfile(self.pickleName): if self.logger != None: self.logger.warning( _("User tried to modify allready filled pickle file.")) self.logger.warning( _("Pickle was named : %s") % self.pickleName) # Creates a new FileStats collector wich spans from the very # start of the hour up till the end. if self.pickleName == "": self.pickleName = StatsPickler.buildThisHoursFileName( client=self.client, currentTime=startTime, machine=self.machine, fileType=fileType) self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\ startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\ interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging ) #Temporarily delete logger to make sure no duplicated lines appears in log file. temp = self.logger del self.logger self.statsCollection.collectStats(endTime) self.logger = temp if save == True: # must remove logger temporarily. Cannot saved opened files. if self.statsCollection.logger != None: temp = self.statsCollection.logger del self.statsCollection.logger loggerNeedsToBeReplaced = True CpickleWrapper.save(object=self.statsCollection, filename=self.pickleName) try: os.chmod(self.pickleName, 0777) dirname = os.path.dirname(self.pickleName) while (dirname != STATSPATHS.STATSPICKLES[:-1] ): #[:-1] removes the last / character try: os.chmod(dirname, 0777) except: pass dirname = os.path.dirname(dirname) except: pass if loggerNeedsToBeReplaced: self.statsCollection.logger = temp if self.logger != None: self.logger.info( _("Saved pickle named : %s ") % self.pickleName)