def prepareQuery(self):
"""
@summary: Buildup the query to be executed.
"""
directory = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, self.queryParameters.machines[0] )
self.queryParameters.sourLients.reverse()
self.graphicProducer = GnuGraphicProducer( directory = directory, fileType = self.queryParameters.fileType,\
clientNames = self.queryParameters.sourLients, \
groupName = self.queryParameters.groupName, \
timespan = int(self.queryParameters.span), currentTime = self.queryParameters.endTime,\
productTypes = self.queryParameters.products, logger= None, logging = False,\
machines = self.queryParameters.machines, workingLanguage = self.queryParameters.language,\
outputLanguage = self.queryParameters.language )
def prepareQuery(self):
"""
@summary: Buildup the query to be executed.
"""
directory = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, self.queryParameters.machines[0])
self.queryParameters.sourLients.reverse()
self.graphicProducer = GnuGraphicProducer( directory = directory, fileType = self.queryParameters.fileType,\
clientNames = self.queryParameters.sourLients, \
groupName = self.queryParameters.groupName, \
timespan = int(self.queryParameters.span), currentTime = self.queryParameters.endTime,\
productTypes = self.queryParameters.products, logger= None, logging = False,\
machines = self.queryParameters.machines, workingLanguage = self.queryParameters.language,\
outputLanguage = self.queryParameters.language )
if usefull == True:
self.entries.append(fileName)
else:
if self.logger != None:
self.logger.warning(
_("Warning in LogFileCollector. Folder named %s does not exist."
) % self.directory)
if __name__ == "__main__":
"""
Small test case. Tests if everything works plus gives an idea on proper usage.
"""
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, LOCAL_MACHINE)
lfc = LogFileCollector(startTime="2006-07-20 01:00:00",
endTime="2006-07-20 02:00:00",
directory=pathToLogFiles,
lastLineRead="",
logType="tx",
name="satnet")
lfc.collectEntries()
print "Files returned : %s " % lfc.entries
def getOptionsFromParser( parser ):
"""
@summary : This method parses the argv received when
the program was called.
It takes the params wich have been passed by
the user and sets them in the corresponding
fields of the infos variable.
@note : If errors are encountered in parameters used,
it will immediatly terminate the application.
"""
currentTime = []
( options, args )= parser.parse_args()
collectUpToNow = options.collectUpToNow
timespan = options.timespan
machines = options.machines.replace( ' ','').split(',')
clientNames = options.clients.replace( ' ','' ).split(',')
types = options.types.replace( ' ', '').split(',')
currentTime = options.currentTime.replace('"','').replace("'",'')
fileType = options.fileType.replace("'",'')
collectUpToNow = options.collectUpToNow
copy = options.copy
combineClients = options.combineClients
productTypes = options.productTypes.replace( ' ', '' ).split( ',' )
groupName = options.groupName.replace( ' ','' )
outputLanguage = options.outputLanguage
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( currentTime, '%Y-%m-%d %H:%M:%S' )
split = currentTime.split()
currentTime = "%s %s" %( split[0], split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS" )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int( timespan ) < 1 :
raise
except:
print _("Error. The timespan value needs to be an integer one above 0." )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if groupName != "" and combineClients == False :
print _("Error. -g|--groupeName option requires the --combineClients options.")
print _('Group names are otherwise irrelevant.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
try :
if fileType == "tx":
validTypes = [ _("errors"), _("filecount"), _("bytecount"), _("latency") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
else:
validTypes = [ _("errors"), _("filecount"), _("bytecount") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
except:
print _("Error. With %s fileType, possible data types values are : %s.") %( fileType,validTypes )
print _('For multiple types use this syntax : -t "type1,type2"')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. %s is not one of the supproted languages")
print _("Use one of the following languages : %s" % str( LanguageTools.getSupportedLanguages() ).replace("[","").replace("]","") )
print _("Use -h for additional help.")
print _("Program terminated.")
clientNames = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters( currentTime, timespan, clientNames, machines, [fileType])
directory = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, machines[0] )
infos = _GraphicsInfos( collectUpToNow = collectUpToNow, currentTime = currentTime, clientNames = clientNames,\
groupName = groupName, directory = directory , types = types, fileType = fileType, \
timespan = timespan, productTypes = productTypes, machines = machines, copy = copy, \
combineClients = combineClients, outputLanguage = outputLanguage )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours( infos.currentTime )
return infos
usefull = self.containsUsefullInfo(fileName)
if usefull == True:
self.entries.append(fileName)
else:
if self.logger != None:
self.logger.warning(_("Warning in LogFileCollector. Folder named %s does not exist.") % self.directory)
if __name__ == "__main__":
"""
Small test case. Tests if everything works plus gives an idea on proper usage.
"""
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles(LOCAL_MACHINE, LOCAL_MACHINE)
lfc = LogFileCollector(
startTime="2006-07-20 01:00:00",
endTime="2006-07-20 02:00:00",
directory=pathToLogFiles,
lastLineRead="",
logType="tx",
name="satnet",
)
lfc.collectEntries()
print "Files returned : %s " % lfc.entries
def getOptionsFromParser(parser):
"""
@summary : This method parses the argv received when
the program was called.
It takes the params wich have been passed by
the user and sets them in the corresponding
fields of the infos variable.
@note : If errors are encountered in parameters used,
it will immediatly terminate the application.
"""
currentTime = []
(options, args) = parser.parse_args()
collectUpToNow = options.collectUpToNow
timespan = options.timespan
machines = options.machines.replace(' ', '').split(',')
clientNames = options.clients.replace(' ', '').split(',')
types = options.types.replace(' ', '').split(',')
currentTime = options.currentTime.replace('"', '').replace("'", '')
fileType = options.fileType.replace("'", '')
collectUpToNow = options.collectUpToNow
copy = options.copy
combineClients = options.combineClients
productTypes = options.productTypes.replace(' ', '').split(',')
groupName = options.groupName.replace(' ', '')
outputLanguage = options.outputLanguage
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime(currentTime, '%Y-%m-%d %H:%M:%S')
split = currentTime.split()
currentTime = "%s %s" % (split[0], split[1])
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int(timespan) < 1:
raise
except:
print _(
"Error. The timespan value needs to be an integer one above 0.")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if groupName != "" and combineClients == False:
print _(
"Error. -g|--groupeName option requires the --combineClients options."
)
print _('Group names are otherwise irrelevant.')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
try:
if fileType == "tx":
validTypes = [
_("errors"),
_("filecount"),
_("bytecount"),
_("latency")
]
if types[0] == _("All"):
types = validTypes
else:
for t in types:
if t not in validTypes:
raise Exception("")
else:
validTypes = [_("errors"), _("filecount"), _("bytecount")]
if types[0] == _("All"):
types = validTypes
else:
for t in types:
if t not in validTypes:
raise Exception("")
except:
print _("Error. With %s fileType, possible data types values are : %s."
) % (fileType, validTypes)
print _('For multiple types use this syntax : -t "type1,type2"')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. %s is not one of the supproted languages")
print _("Use one of the following languages : %s" %
str(LanguageTools.getSupportedLanguages()).replace(
"[", "").replace("]", ""))
print _("Use -h for additional help.")
print _("Program terminated.")
clientNames = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
currentTime, timespan, clientNames, machines, [fileType])
directory = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, machines[0])
infos = _GraphicsInfos( collectUpToNow = collectUpToNow, currentTime = currentTime, clientNames = clientNames,\
groupName = groupName, directory = directory , types = types, fileType = fileType, \
timespan = timespan, productTypes = productTypes, machines = machines, copy = copy, \
combineClients = combineClients, outputLanguage = outputLanguage )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours(infos.currentTime)
return infos
def updateHourlyPickles( infos, paths, logger = None ):
"""
@summary : This method is to be used when hourly pickling is done. -1 pickle per hour per client.
This method needs will update the pickles by collecting data from the time of the last
pickle up to the current date.(System time or the one specified by the user.)
If for some reason data wasnt collected for one or more hour since last pickle,pickles
for the missing hours will be created and filled with data.
If no entries are found for this client in the pickled-times file, we take for granted that
this is a new client. In that case data will be collected from the top of the hour up to the
time of the call.
If new client has been producing data before the day of the first call, user can specify a
different time than system time to specify the first day to pickle. He can then call this
method with the current system time, and data between first day and current time will be
collected so that pickling can continue like the other clients can.
"""
sp = StatsPickler( logger = logger )
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, infos.machine )
for i in range( len (infos.clients) ) :
sp.client = infos.clients[i]
width = StatsDateLib.getSecondsSinceEpoch(infos.endTime) - StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoWithRoundedHours(infos.startTimes[i] ) )
if width > StatsDateLib.HOUR :#In case pickling didnt happen for a few hours for some reason...
hours = [infos.startTimes[i]]
hours.extend( StatsDateLib.getSeparatorsWithStartTime( infos.startTimes[i], interval = StatsDateLib.HOUR, width = width ))
for j in range( len(hours)-1 ): #Covers hours where no pickling was done.
startOfTheHour = StatsDateLib.getIsoWithRoundedHours( hours[j] )
startTime = startOfTheHour
endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoWithRoundedHours(hours[j+1] ) ))
#print " client : %s startTime : %s endTime : %s" %(infos.clients[i], startTime, endTime )
if startTime >= endTime and logger != None :
try:
logger.warning( _("Startime used in updateHourlyPickles was greater or equal to end time.") )
except:
pass
sp.pickleName = StatsPickler.buildThisHoursFileName( client = infos.clients[i], currentTime = startOfTheHour, machine = infos.machine, fileType = infos.fileType )
sp.collectStats( types = infos.types, startTime = startTime , endTime = endTime, interval = infos.interval * StatsDateLib.MINUTE,\
directory = pathToLogFiles, fileType = infos.fileType )
else:
startTime = infos.startTimes[i]
endTime = infos.endTime
startOfTheHour = StatsDateLib.getIsoWithRoundedHours( infos.startTimes[i] )
#print " client : %s startTime : %s endTime : %s" %(infos.clients[i], startTime, endTime )
if startTime >= endTime and logger != None :#to be removed
try:
logger.warning( _("Startime used in updateHourlyPickles was greater or equal to end time.") )
except:
pass
sp.pickleName = StatsPickler.buildThisHoursFileName( client = infos.clients[i], currentTime = startOfTheHour, machine = infos.machine, fileType = infos.fileType )
sp.collectStats( infos.types, startTime = startTime, endTime = endTime, interval = infos.interval * StatsDateLib.MINUTE, directory = pathToLogFiles, fileType = infos.fileType )
setLastUpdate( machine = infos.machine, client = infos.clients[i], fileType = infos.fileType, currentDate = infos.currentDate, paths = paths, collectUpToNow = infos.collectUpToNow )
def getOptionsFromParser( parser, paths, logger = None ):
"""
@summary : This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the hlE variable.
@Warning : If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
directories = []
startTimes = []
( options, args ) = parser.parse_args()
interval = options.interval
collectUpToNow = options.collectUpToNow
currentDate = options.currentDate.replace( '"','' ).replace( "'",'' )
currentDate = StatsDateLib.getIsoWithRoundedHours( currentDate )
fileType = options.fileType.replace( "'",'' )
machine = options.machine.replace( " ","" )
clients = options.clients.replace(' ','' ).split( ',' )
types = options.types.replace( ' ', '' ).split( ',' )
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, machine )
#print "*****pathToLogFiles %s" %pathToLogFiles
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( currentDate, '%Y-%m-%d %H:%M:%S' )
split = currentDate.split()
currentDate = "%s %s" %( split[0],split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS" )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int( interval ) < 1 :
raise
except:
print _("Error. The interval value needs to be an integer one above 0." )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if fileType == "tx":
validTypes = [ _("errors"), _("latency"), _("bytecount") ]
else:
validTypes = [ _("errors"), _("bytecount") ]
if types[0] == _("All"):
types = validTypes
try :
for t in types :
if t not in validTypes:
raise
except:
print _("Error. With %s fileType, possible data types values are : %s.") %(fileType,validTypes )
print _('For multiple types use this syntax : -t "type1,type2"' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
def translateType(typeToTranslate):
translations = { _("errors"):"errors", _("latency"):"latency", _("bytecount"):"bytecount" }
return translations[typeToTranslate]
types = map( translateType, types )
if clients[0] == _("All") :
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine )
if fileType == "tx":
clients = txNames
else:
clients = rxNames
#print "clients found :%s" %clients
# Verify that each client needs to be updated.
# If not we add a warning to the logger and removwe the client from the list
# since it's not needed, but other clients might be.
usefullClients = []
for client in clients :
startTime = getLastUpdate( machine = machine, client = client, fileType= fileType, currentDate = currentDate , paths = paths, collectUpToNow = collectUpToNow )
if currentDate > startTime:
#print " client : %s currentDate : %s startTime : %s" %( client, currentDate, startTime )
directories.append( pathToLogFiles )
startTimes.append( startTime )
usefullClients.append( client )
else:
#print "This client was not updated since it's last update was more recent than specified date : %s" %client
if logger != None :
try:
logger.warning( _("This client was not updated since it's last update was more recent than specified date : %s") %client)
except :
pass
infos = _UpdaterInfos( currentDate = currentDate, clients = usefullClients, startTimes = startTimes, directories = directories ,\
types = types, collectUpToNow = collectUpToNow, fileType = fileType, machine = machine )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours( infos.currentDate )
return infos