def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\
pickleName = "", logger = None, logging = False, machine = "pdsCSP" ):
"""
Constructor.
-Builds a StatsPickler with no entries.
"""
self.client = client # Name of the client for whom were collecting data
self.pickleName = "" # Pickle
self.directory = directory # Name of the directory containing stats files.
self.statsTypes = statsTypes or [] # Types we'll search for stats.
self.machine = machine # Machine on wich the data resides.
self.loggerName = 'pickling' # Name of the logger.
self.logger = logger # Permits a logging system for this object.
self.logging = logging # Whether or not to enable logging.
if self.logging == True:
if logger is None: # Enable logging
if not os.path.isdir( STATSPATHS.STATSLOGGING ):
os.makedirs( STATSPATHS.STATSLOGGING , mode=0777 )
self.logger = Logger( STATSPATHS.STATSLOGGING + 'stats_' + self.loggerName + '.log.notb', 'INFO', 'TX' + self.loggerName, bytes = True )
self.logger = self.logger.getLogger()
else:
logger = True
self.statsCollection = statsCollection or FileStatsCollector( logger = self.logger, logging = logging )
self.fileCollection = LogFileCollector( directory = directory, logger = self.logger, logging = logging )
global _
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\
pickleName = "", logger = None, logging = False, machine = "pdsCSP" ):
"""
Constructor.
-Builds a StatsPickler with no entries.
"""
self.client = client # Name of the client for whom were collecting data
self.pickleName = "" # Pickle
self.directory = directory # Name of the directory containing stats files.
self.statsTypes = statsTypes or [] # Types we'll search for stats.
self.machine = machine # Machine on wich the data resides.
self.loggerName = 'pickling' # Name of the logger.
self.logger = logger # Permits a logging system for this object.
self.logging = logging # Whether or not to enable logging.
if self.logging == True:
if logger is None: # Enable logging
if not os.path.isdir(STATSPATHS.STATSLOGGING):
os.makedirs(STATSPATHS.STATSLOGGING, mode=0777)
self.logger = Logger(STATSPATHS.STATSLOGGING + 'stats_' +
self.loggerName + '.log.notb',
'INFO',
'TX' + self.loggerName,
bytes=True)
self.logger = self.logger.getLogger()
else:
logger = True
self.statsCollection = statsCollection or FileStatsCollector(
logger=self.logger, logging=logging)
self.fileCollection = LogFileCollector(directory=directory,
logger=self.logger,
logging=logging)
global _
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
class StatsPickler(Translatable):
"""
Contains all the methods needed to pickle stats for a certain client.
"""
def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\
pickleName = "", logger = None, logging = False, machine = "pdsCSP" ):
"""
Constructor.
-Builds a StatsPickler with no entries.
"""
self.client = client # Name of the client for whom were collecting data
self.pickleName = "" # Pickle
self.directory = directory # Name of the directory containing stats files.
self.statsTypes = statsTypes or [] # Types we'll search for stats.
self.machine = machine # Machine on wich the data resides.
self.loggerName = 'pickling' # Name of the logger.
self.logger = logger # Permits a logging system for this object.
self.logging = logging # Whether or not to enable logging.
if self.logging == True:
if logger is None: # Enable logging
if not os.path.isdir(STATSPATHS.STATSLOGGING):
os.makedirs(STATSPATHS.STATSLOGGING, mode=0777)
self.logger = Logger(STATSPATHS.STATSLOGGING + 'stats_' +
self.loggerName + '.log.notb',
'INFO',
'TX' + self.loggerName,
bytes=True)
self.logger = self.logger.getLogger()
else:
logger = True
self.statsCollection = statsCollection or FileStatsCollector(
logger=self.logger, logging=logging)
self.fileCollection = LogFileCollector(directory=directory,
logger=self.logger,
logging=logging)
global _
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
def buildThisHoursFileName(client="someclient",
offset=0,
currentTime="",
fileType="tx",
machine="someMachineName"):
"""
@summary : Builds a filename using current currentTime.
@Note : The format will be something like this :
StatsPaths.STATSPICKLES/clientName/date/TXorRX//machine_hour
Ex : StatsPaths.STATSPICKLES/clientName/20060707/tx/machinex_12:00:00
offset can be used to find a file from an hour close to the current one
tempcurrentTime can also be used to build a filename from another hour.
@warning :To be used only with pickles created hourly.
"""
timeFolder = ""
if currentTime == "":
currentTime = time.time()
else:
currentTime = StatsDateLib.getSecondsSinceEpoch(currentTime)
currentTime = currentTime + (offset * StatsDateLib.HOUR)
splitTime = time.gmtime(currentTime)
for i in range(3):
if int(splitTime[i]) < 10:
timeFolder = timeFolder + "0" + str(splitTime[i])
else:
timeFolder = timeFolder + str(splitTime[i])
hour = StatsDateLib.getHoursFromIso(
StatsDateLib.getIsoFromEpoch(currentTime))
maxLt = (os.statvfs(STATSPATHS.STATSPICKLES)[statvfs.F_NAMEMAX])
fileName = ("%s" + "%." + str(maxLt) + "s/%s/%s/%." + str(maxLt) +
"s_%s") % (STATSPATHS.STATSPICKLES, client, timeFolder,
fileType, str(machine), str(hour))
return fileName
buildThisHoursFileName = staticmethod(buildThisHoursFileName)
def collectStats(self,
types,
directory,
fileType="tx",
startTime='2006-05-18 00:00:00',
endTime="",
interval=60 * StatsDateLib.MINUTE,
save=True):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger #Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None: #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile(self.pickleName):
if self.logger != None:
self.logger.warning(
_("User tried to modify allready filled pickle file."))
self.logger.warning(
_("Pickle was named : %s") % self.pickleName)
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName(
client=self.client,
currentTime=startTime,
machine=self.machine,
fileType=fileType)
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats(endTime)
self.logger = temp
if save == True: # must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save(object=self.statsCollection,
filename=self.pickleName)
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname(self.pickleName)
while (dirname != STATSPATHS.STATSPICKLES[:-1]
): #[:-1] removes the last / character
try:
os.chmod(dirname, 0777)
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced:
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info(
_("Saved pickle named : %s ") % self.pickleName)
def printStats(self):
"""
@summary : This utility method prints out all the stats concerning each files.
@note : ****Mostly usefull for debugging****
"""
global _
absoluteFilename = str(
STATSPATHS.STATSDATA) + "TEST_OUTPUT_FILE_FOR_STATSPICKLER "
print _("Output filename used : %s") % absoluteFilename
fileHandle = open(absoluteFilename, 'w')
old_stdout = sys.stdout
sys.stdout = fileHandle
print _("\n\nFiles used : %s") % self.fileCollection.entries
print _("Starting date: %s") % self.statsCollection.startTime
print "Interval: %s" % self.statsCollection.interval
print "endTime: %s" % self.statsCollection.endTime
for j in range(self.statsCollection.nbEntries):
print _("\nEntry's interval : %s - %s ") % (
self.statsCollection.fileEntries[j].startTime,
self.statsCollection.fileEntries[j].endTime)
print _("Values :")
print self.statsCollection.fileEntries[j].values.dictionary
print _("Means :")
print self.statsCollection.fileEntries[j].means
print _("Medians")
print self.statsCollection.fileEntries[j].medians
print _("Minimums")
print self.statsCollection.fileEntries[j].minimums
print _("Maximums")
print self.statsCollection.fileEntries[j].maximums
print _("Total")
print self.statsCollection.fileEntries[j].totals
fileHandle.close()
sys.stdout = old_stdout #resets standard output
print _("Printed %s ") % absoluteFilename
def collectStats(self,
types,
directory,
fileType="tx",
startTime='2006-05-18 00:00:00',
endTime="",
interval=60 * StatsDateLib.MINUTE,
save=True):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger #Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None: #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile(self.pickleName):
if self.logger != None:
self.logger.warning(
_("User tried to modify allready filled pickle file."))
self.logger.warning(
_("Pickle was named : %s") % self.pickleName)
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName(
client=self.client,
currentTime=startTime,
machine=self.machine,
fileType=fileType)
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats(endTime)
self.logger = temp
if save == True: # must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save(object=self.statsCollection,
filename=self.pickleName)
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname(self.pickleName)
while (dirname != STATSPATHS.STATSPICKLES[:-1]
): #[:-1] removes the last / character
try:
os.chmod(dirname, 0777)
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced:
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info(
_("Saved pickle named : %s ") % self.pickleName)
class StatsPickler(Translatable):
"""
Contains all the methods needed to pickle stats for a certain client.
"""
def __init__( self, client = "", directory = "", statsTypes = None, statsCollection = None,\
pickleName = "", logger = None, logging = False, machine = "pdsCSP" ):
"""
Constructor.
-Builds a StatsPickler with no entries.
"""
self.client = client # Name of the client for whom were collecting data
self.pickleName = "" # Pickle
self.directory = directory # Name of the directory containing stats files.
self.statsTypes = statsTypes or [] # Types we'll search for stats.
self.machine = machine # Machine on wich the data resides.
self.loggerName = 'pickling' # Name of the logger.
self.logger = logger # Permits a logging system for this object.
self.logging = logging # Whether or not to enable logging.
if self.logging == True:
if logger is None: # Enable logging
if not os.path.isdir( STATSPATHS.STATSLOGGING ):
os.makedirs( STATSPATHS.STATSLOGGING , mode=0777 )
self.logger = Logger( STATSPATHS.STATSLOGGING + 'stats_' + self.loggerName + '.log.notb', 'INFO', 'TX' + self.loggerName, bytes = True )
self.logger = self.logger.getLogger()
else:
logger = True
self.statsCollection = statsCollection or FileStatsCollector( logger = self.logger, logging = logging )
self.fileCollection = LogFileCollector( directory = directory, logger = self.logger, logging = logging )
global _
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH)
def buildThisHoursFileName( client = "someclient", offset = 0, currentTime = "", fileType = "tx", machine = "someMachineName" ):
"""
@summary : Builds a filename using current currentTime.
@Note : The format will be something like this :
StatsPaths.STATSPICKLES/clientName/date/TXorRX//machine_hour
Ex : StatsPaths.STATSPICKLES/clientName/20060707/tx/machinex_12:00:00
offset can be used to find a file from an hour close to the current one
tempcurrentTime can also be used to build a filename from another hour.
@warning :To be used only with pickles created hourly.
"""
timeFolder = ""
if currentTime == "":
currentTime = time.time()
else:
currentTime = StatsDateLib.getSecondsSinceEpoch( currentTime )
currentTime = currentTime + ( offset * StatsDateLib.HOUR )
splitTime = time.gmtime( currentTime )
for i in range( 3 ):
if int( splitTime[i] ) < 10 :
timeFolder = timeFolder + "0" + str( splitTime[i] )
else:
timeFolder = timeFolder + str( splitTime[i] )
hour = StatsDateLib.getHoursFromIso( StatsDateLib.getIsoFromEpoch( currentTime ) )
maxLt = ( os.statvfs( STATSPATHS.STATSPICKLES )[statvfs.F_NAMEMAX])
fileName = ( "%s" + "%." + str( maxLt ) + "s/%s/%s/%." + str( maxLt ) + "s_%s" ) %( STATSPATHS.STATSPICKLES, client, timeFolder, fileType, str(machine), str(hour) )
return fileName
buildThisHoursFileName = staticmethod( buildThisHoursFileName )
def collectStats( self, types, directory, fileType = "tx", startTime = '2006-05-18 00:00:00', endTime = "", interval = 60*StatsDateLib.MINUTE, save = True ):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger#Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None : #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile( self.pickleName ):
if self.logger != None :
self.logger.warning( _("User tried to modify allready filled pickle file." ) )
self.logger.warning( _("Pickle was named : %s") %self.pickleName )
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName( client = self.client, currentTime = startTime, machine = self.machine, fileType = fileType )
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats( endTime )
self.logger = temp
if save == True :# must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save ( object = self.statsCollection, filename = self.pickleName )
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname( self.pickleName )
while( dirname != STATSPATHS.STATSPICKLES[:-1] ):#[:-1] removes the last / character
try:
os.chmod( dirname, 0777 )
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced :
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info( _("Saved pickle named : %s ") %self.pickleName )
def printStats( self ) :
"""
@summary : This utility method prints out all the stats concerning each files.
@note : ****Mostly usefull for debugging****
"""
global _
absoluteFilename = str( STATSPATHS.STATSDATA ) + "TEST_OUTPUT_FILE_FOR_STATSPICKLER "
print _("Output filename used : %s") %absoluteFilename
fileHandle = open( absoluteFilename , 'w' )
old_stdout = sys.stdout
sys.stdout = fileHandle
print _("\n\nFiles used : %s") %self.fileCollection.entries
print _("Starting date: %s") % self.statsCollection.startTime
print "Interval: %s" %self.statsCollection.interval
print "endTime: %s" %self.statsCollection.endTime
for j in range( self.statsCollection.nbEntries ):
print _("\nEntry's interval : %s - %s ") %(self.statsCollection.fileEntries[j].startTime,self.statsCollection.fileEntries[j].endTime)
print _("Values :")
print self.statsCollection.fileEntries[j].values.dictionary
print _("Means :")
print self.statsCollection.fileEntries[j].means
print _("Medians" )
print self.statsCollection.fileEntries[j].medians
print _("Minimums")
print self.statsCollection.fileEntries[j].minimums
print _("Maximums")
print self.statsCollection.fileEntries[j].maximums
print _("Total")
print self.statsCollection.fileEntries[j].totals
fileHandle.close()
sys.stdout = old_stdout #resets standard output
print _("Printed %s ") %absoluteFilename
def collectStats( self, types, directory, fileType = "tx", startTime = '2006-05-18 00:00:00', endTime = "", interval = 60*StatsDateLib.MINUTE, save = True ):
"""
@summary : This method is used to collect stats from logfiles found within a directory.
Types is the type of dats to be collected.
Pickle is the name of the file to be used. If not specified will be generated
according to the other parameters.
FileType specifies what type of files to look for in the directory.
StartTime and endtime specify the boundaries within wich we'll collect the data.
Interval the width of the entries in the stats collection
save can be false if for some reason user does not want to save pickle.
If both of the above are true, hourly pickles will be done.
Pre-conditions : StarTime needs to be smaller than endTime.
If Daily pickling is used,width between start
and endTime needs to be no more than 24hours
If Hourly pickling is used,width between start
and endTime needs to be no more than 1hour.
If pre-conditions aren't met, application will fail.
"""
global _
#Find up to date file list.
self.fileCollection = LogFileCollector( startTime = startTime , endTime = endTime, directory = directory, lastLineRead = "",\
logType = fileType, name = self.client, logger = self.logger, logging = self.logging )
temp = self.logger#Need to remove current logger temporarily
del self.logger
self.fileCollection.collectEntries() #find all entries from the folder
self.logger = temp
if self.fileCollection.logger != None : #No longer need the logger
self.fileCollection.logger = None
if os.path.isfile( self.pickleName ):
if self.logger != None :
self.logger.warning( _("User tried to modify allready filled pickle file." ) )
self.logger.warning( _("Pickle was named : %s") %self.pickleName )
# Creates a new FileStats collector wich spans from the very
# start of the hour up till the end.
if self.pickleName == "":
self.pickleName = StatsPickler.buildThisHoursFileName( client = self.client, currentTime = startTime, machine = self.machine, fileType = fileType )
self.statsCollection = FileStatsCollector( files = self.fileCollection.entries, fileType = fileType, statsTypes = types,\
startTime = StatsDateLib.getIsoWithRoundedHours( startTime ), endTime = endTime,\
interval = interval, totalWidth = 1*StatsDateLib.HOUR, logger = self.logger,logging = self.logging )
#Temporarily delete logger to make sure no duplicated lines appears in log file.
temp = self.logger
del self.logger
self.statsCollection.collectStats( endTime )
self.logger = temp
if save == True :# must remove logger temporarily. Cannot saved opened files.
if self.statsCollection.logger != None:
temp = self.statsCollection.logger
del self.statsCollection.logger
loggerNeedsToBeReplaced = True
CpickleWrapper.save ( object = self.statsCollection, filename = self.pickleName )
try:
os.chmod(self.pickleName, 0777)
dirname = os.path.dirname( self.pickleName )
while( dirname != STATSPATHS.STATSPICKLES[:-1] ):#[:-1] removes the last / character
try:
os.chmod( dirname, 0777 )
except:
pass
dirname = os.path.dirname(dirname)
except:
pass
if loggerNeedsToBeReplaced :
self.statsCollection.logger = temp
if self.logger != None:
self.logger.info( _("Saved pickle named : %s ") %self.pickleName )