def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\
pickleName = "", logger = None, logging = False, machine = "pdsCSP" ):
"""
Constructor.
-Builds a StatsPickler with no entries.
"""
self.client = client # Name of the client for whom were collecting data
self.pickleName = "" # Pickle
self.directory = directory # Name of the directory containing stats files.
self.statsTypes = statsTypes or [] # Types we'll search for stats.
self.machine = machine # Machine on wich the data resides.
self.loggerName = 'pickling' # Name of the logger.
self.logger = logger # Permits a logging system for this object.
self.logging = logging # Whether or not to enable logging.
if self.logging == True:
if logger is None: # Enable logging
if not os.path.isdir(STATSPATHS.STATSLOGGING):
os.makedirs(STATSPATHS.STATSLOGGING, mode=0777)
self.logger = Logger(STATSPATHS.STATSLOGGING + 'stats_' +
self.loggerName + '.log.notb',
'INFO',
'TX' + self.loggerName,
bytes=True)
self.logger = self.logger.getLogger()
else:
logger = True
self.statsCollection = statsCollection or FileStatsCollector(
logger=self.logger, logging=logging)
self.fileCollection = LogFileCollector(directory=directory,
logger=self.logger,
logging=logging)
global _
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
def collectStats(self,
types,
directory,
fileType="tx",
startTime='2006-05-18 00:00:00',
endTime="",
interval=60 * StatsDateLib.MINUTE,
save=True):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger #Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None: #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile(self.pickleName):
if self.logger != None:
self.logger.warning(
_("User tried to modify allready filled pickle file."))
self.logger.warning(
_("Pickle was named : %s") % self.pickleName)
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName(
client=self.client,
currentTime=startTime,
machine=self.machine,
fileType=fileType)
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats(endTime)
self.logger = temp
if save == True: # must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save(object=self.statsCollection,
filename=self.pickleName)
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname(self.pickleName)
while (dirname != STATSPATHS.STATSPICKLES[:-1]
): #[:-1] removes the last / character
try:
os.chmod(dirname, 0777)
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced:
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info(
_("Saved pickle named : %s ") % self.pickleName)
