def getPairs(clients,
machines,
fileType,
startTime,
endTime,
groupName="",
logger=None):
"""
@summary : This method gathers all the data pairs needed to update the different
databases associated with a client of a certain fileType.
"""
dataPairs = {}
dataTypes = GeneralStatsLibraryMethods.getDataTypesAssociatedWithFileType(
fileType)
mergedData = getMergedData(clients, fileType, machines, startTime, endTime,
groupName, logger)
for dataType in dataTypes:
dataPairs[dataType] = getPairsFromMergedData(dataType, mergedData,
logger)
return dataPairs
def isInterestingProduct( product = "", interestingProductTypes = ["All"] ):
'''
@param product: Product to verifry.
@param interestingProductTypes: Array containing the list of valid product types to look for.
Product types can contain wildcard characters.
'''
isInterestingProduct = False
#print "produt : %s interested in : %s " %(product , interestingProductTypes )
for productType in interestingProductTypes:
if productType == "All":
isInterestingProduct = True
break
else:
pattern = GeneralStatsLibraryMethods.buildPattern(productType)
if fnmatch.fnmatch( product, pattern) == True:
isInterestingProduct = True
break
return isInterestingProduct
def generateGraphsForIndividualMachines(infos):
"""
@summary : Generate graphs for every specified machine withoout
merging any of the data between the machines.
"""
for i in range(len(infos.machines)):
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, infos.machines[i])
j = 0
for txName in txNames:
pid = os.fork() #create child process
if pid == 0: #child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m '%s' -f tx -c '%s' -d '%s' -s %s --copy --outputLanguage %s" \
%( STATSPATHS.STATSBIN, infos.machines[i], txName, infos.date, infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j % 10 == 0:
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
j = 0
for rxName in rxNames:
pid = os.fork() #create child process
if pid == 0: #child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m '%s' -f rx -c '%s' -d '%s' -s %s --copy --outputLanguage %s"\
%( STATSPATHS.STATSBIN, infos.machines[i] , rxName, infos.date,infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j % 10 == 0:
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
def generateGraphsForIndividualMachines( infos ) :
"""
@summary : Generate graphs for every specified machine withoout
merging any of the data between the machines.
"""
for i in range ( len( infos.machines ) ) :
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, infos.machines[i] )
j=0
for txName in txNames :
pid = os.fork()#create child process
if pid == 0: #child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m '%s' -f tx -c '%s' -d '%s' -s %s --copy --outputLanguage %s" \
%( STATSPATHS.STATSBIN, infos.machines[i], txName, infos.date, infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j %10 == 0:
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
j=0
for rxName in rxNames:
pid = os.fork()#create child process
if pid == 0 :#child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m '%s' -f rx -c '%s' -d '%s' -s %s --copy --outputLanguage %s"\
%( STATSPATHS.STATSBIN, infos.machines[i] , rxName, infos.date,infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j %10 == 0:
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
def generateWebPage(self):
"""
@summary : Call to generate the web page.
"""
self.setYears()
start, end = self.getStartEndOfWebPage()
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesForWebPages(start, end)
self.printWebPage(rxNames, txNames)
def generateWebPage( self ):
"""
@summary : Call to generate the web page.
"""
self.setYears()
start, end = self.getStartEndOfWebPage()
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesForWebPages( start, end )
self.printWebPage( rxNames, txNames )
def prepareQuery(self):
"""
@summary: Buildup the query to be executed.
"""
directory = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, self.queryParameters.machines[0] )
self.queryParameters.sourLients.reverse()
self.graphicProducer = GnuGraphicProducer( directory = directory, fileType = self.queryParameters.fileType,\
clientNames = self.queryParameters.sourLients, \
groupName = self.queryParameters.groupName, \
timespan = int(self.queryParameters.span), currentTime = self.queryParameters.endTime,\
productTypes = self.queryParameters.products, logger= None, logging = False,\
machines = self.queryParameters.machines, workingLanguage = self.queryParameters.language,\
outputLanguage = self.queryParameters.language )
def prepareQuery(self):
"""
@summary: Buildup the query to be executed.
"""
directory = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, self.queryParameters.machines[0])
self.queryParameters.sourLients.reverse()
self.graphicProducer = GnuGraphicProducer( directory = directory, fileType = self.queryParameters.fileType,\
clientNames = self.queryParameters.sourLients, \
groupName = self.queryParameters.groupName, \
timespan = int(self.queryParameters.span), currentTime = self.queryParameters.endTime,\
productTypes = self.queryParameters.products, logger= None, logging = False,\
machines = self.queryParameters.machines, workingLanguage = self.queryParameters.language,\
outputLanguage = self.queryParameters.language )
def getPairs( clients, machines, fileType, startTime, endTime, groupName = "", logger = None ):
"""
@summary : This method gathers all the data pairs needed to update the different
databases associated with a client of a certain fileType.
"""
dataPairs = {}
dataTypes = GeneralStatsLibraryMethods.getDataTypesAssociatedWithFileType(fileType)
mergedData = getMergedData( clients, fileType, machines, startTime, endTime, groupName, logger )
for dataType in dataTypes :
dataPairs[ dataType ] = getPairsFromMergedData( dataType, mergedData, logger )
return dataPairs
def getAllClientOrSourcesNamesFromMachines(infos):
"""
@summary : Goes through all the machines and finds out
wich client or sources currently run on each
of those machines.
To make sure no confusion arrises if to clinets
or source have the same name on different
machhines or cluster, the returned names will
be associated with all the machines/clusters
with whom they are associated as to let the caller
hadnle the situation as it pleases.
@param infos: Infos that were gathered at program call.
@return : The dictionary containing the names and their associated machines.
"""
sourlients = {}
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith(machine)
if machines != []:
#print machines
machine = str(machines).replace('[', '').replace(
']',
'').replace(',',
'').replace("'",
'').replace('"',
'').replace(" ", '')
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end,
machines,
pattern=None,
havingrunOnAllMachines=True)
else:
rxNames, txNames = [], []
else: #might be a groups machine,in this case do not force to have run on all machines.....
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end, [machine],
pattern=None,
havingrunOnAllMachines=False)
else:
rxNames, txNames = [], []
else: #not a cluster, une mahcine name directly. Force to have run exclusivly on specified machine.
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
infos.start,
infos.end, [machine],
pattern=None,
havingrunOnAllMachines=True)
else:
rxNames, txNames = [], []
if infos.fileType == "rx":
namesToAdd = rxNames
elif infos.fileType == "tx":
namesToAdd = txNames
for nameToAdd in namesToAdd:
if nameToAdd in sourlients.keys():
if machine not in sourlients[nameToAdd]:
sourlients[nameToAdd].append(machine)
else:
sourlients[nameToAdd] = [machine]
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
for group in groups:
if configParameters.groupParameters.groupFileTypes[
group] == infos.fileType:
sourlients[group] = [str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')]
return sourlients
def getAllClientOrSourcesNamesFromMachines( infos ):
"""
@summary : Goes through all the machines and finds out
wich client or sources currently run on each
of those machines.
To make sure no confusion arrises if to clinets
or source have the same name on different
machhines or cluster, the returned names will
be associated with all the machines/clusters
with whom they are associated as to let the caller
hadnle the situation as it pleases.
@param infos: Infos that were gathered at program call.
@return : The dictionary containing the names and their associated machines.
"""
sourlients ={}
for machine in infos.machinesToSearch:
if infos.machinesAreClusters == True:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
machines = machineConfig.getMachinesAssociatedWith( machine )
if machines != []:
#print machines
machine = str( machines ).replace('[','').replace(']', '').replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, machines, pattern = None, havingrunOnAllMachines = True )
else:
rxNames, txNames = [],[]
else: #might be a groups machine,in this case do not force to have run on all machines.....
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, [machine], pattern = None, havingrunOnAllMachines = False )
else:
rxNames, txNames = [],[]
else:#not a cluster, une mahcine name directly. Force to have run exclusivly on specified machine.
if machine != '':
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( infos.start, infos.end, [machine], pattern = None, havingrunOnAllMachines = True )
else:
rxNames, txNames = [],[]
if infos.fileType == "rx":
namesToAdd = rxNames
elif infos.fileType == "tx":
namesToAdd = txNames
for nameToAdd in namesToAdd:
if nameToAdd in sourlients.keys():
if machine not in sourlients[nameToAdd]:
sourlients[nameToAdd].append( machine )
else:
sourlients[nameToAdd] = [ machine ]
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
groups = configParameters.groupParameters.groups
for group in groups :
if configParameters.groupParameters.groupFileTypes[group] == infos.fileType :
sourlients[group] = [str( configParameters.groupParameters.groupsMachines[group] ).replace('[','').replace(']', '').\
replace(',','').replace( "'",'' ).replace('"','' ).replace(" ",'')]
return sourlients
def getOptionsFromParser(parser):
"""
@summary : This method parses the argv received when
the program was called.
It takes the params wich have been passed by
the user and sets them in the corresponding
fields of the infos variable.
@note : If errors are encountered in parameters used,
it will immediatly terminate the application.
"""
currentTime = []
(options, args) = parser.parse_args()
collectUpToNow = options.collectUpToNow
timespan = options.timespan
machines = options.machines.replace(' ', '').split(',')
clientNames = options.clients.replace(' ', '').split(',')
types = options.types.replace(' ', '').split(',')
currentTime = options.currentTime.replace('"', '').replace("'", '')
fileType = options.fileType.replace("'", '')
collectUpToNow = options.collectUpToNow
copy = options.copy
combineClients = options.combineClients
productTypes = options.productTypes.replace(' ', '').split(',')
groupName = options.groupName.replace(' ', '')
outputLanguage = options.outputLanguage
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime(currentTime, '%Y-%m-%d %H:%M:%S')
split = currentTime.split()
currentTime = "%s %s" % (split[0], split[1])
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int(timespan) < 1:
raise
except:
print _(
"Error. The timespan value needs to be an integer one above 0.")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if groupName != "" and combineClients == False:
print _(
"Error. -g|--groupeName option requires the --combineClients options."
)
print _('Group names are otherwise irrelevant.')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
try:
if fileType == "tx":
validTypes = [
_("errors"),
_("filecount"),
_("bytecount"),
_("latency")
]
if types[0] == _("All"):
types = validTypes
else:
for t in types:
if t not in validTypes:
raise Exception("")
else:
validTypes = [_("errors"), _("filecount"), _("bytecount")]
if types[0] == _("All"):
types = validTypes
else:
for t in types:
if t not in validTypes:
raise Exception("")
except:
print _("Error. With %s fileType, possible data types values are : %s."
) % (fileType, validTypes)
print _('For multiple types use this syntax : -t "type1,type2"')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. %s is not one of the supproted languages")
print _("Use one of the following languages : %s" %
str(LanguageTools.getSupportedLanguages()).replace(
"[", "").replace("]", ""))
print _("Use -h for additional help.")
print _("Program terminated.")
clientNames = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
currentTime, timespan, clientNames, machines, [fileType])
directory = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, machines[0])
infos = _GraphicsInfos( collectUpToNow = collectUpToNow, currentTime = currentTime, clientNames = clientNames,\
groupName = groupName, directory = directory , types = types, fileType = fileType, \
timespan = timespan, productTypes = productTypes, machines = machines, copy = copy, \
combineClients = combineClients, outputLanguage = outputLanguage )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours(infos.currentTime)
return infos
def updateHourlyPickles( infos, paths, logger = None ):
"""
@summary : This method is to be used when hourly pickling is done. -1 pickle per hour per client.
This method needs will update the pickles by collecting data from the time of the last
pickle up to the current date.(System time or the one specified by the user.)
If for some reason data wasnt collected for one or more hour since last pickle,pickles
for the missing hours will be created and filled with data.
If no entries are found for this client in the pickled-times file, we take for granted that
this is a new client. In that case data will be collected from the top of the hour up to the
time of the call.
If new client has been producing data before the day of the first call, user can specify a
different time than system time to specify the first day to pickle. He can then call this
method with the current system time, and data between first day and current time will be
collected so that pickling can continue like the other clients can.
"""
sp = StatsPickler( logger = logger )
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, infos.machine )
for i in range( len (infos.clients) ) :
sp.client = infos.clients[i]
width = StatsDateLib.getSecondsSinceEpoch(infos.endTime) - StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoWithRoundedHours(infos.startTimes[i] ) )
if width > StatsDateLib.HOUR :#In case pickling didnt happen for a few hours for some reason...
hours = [infos.startTimes[i]]
hours.extend( StatsDateLib.getSeparatorsWithStartTime( infos.startTimes[i], interval = StatsDateLib.HOUR, width = width ))
for j in range( len(hours)-1 ): #Covers hours where no pickling was done.
startOfTheHour = StatsDateLib.getIsoWithRoundedHours( hours[j] )
startTime = startOfTheHour
endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( StatsDateLib.getIsoWithRoundedHours(hours[j+1] ) ))
#print " client : %s startTime : %s endTime : %s" %(infos.clients[i], startTime, endTime )
if startTime >= endTime and logger != None :
try:
logger.warning( _("Startime used in updateHourlyPickles was greater or equal to end time.") )
except:
pass
sp.pickleName = StatsPickler.buildThisHoursFileName( client = infos.clients[i], currentTime = startOfTheHour, machine = infos.machine, fileType = infos.fileType )
sp.collectStats( types = infos.types, startTime = startTime , endTime = endTime, interval = infos.interval * StatsDateLib.MINUTE,\
directory = pathToLogFiles, fileType = infos.fileType )
else:
startTime = infos.startTimes[i]
endTime = infos.endTime
startOfTheHour = StatsDateLib.getIsoWithRoundedHours( infos.startTimes[i] )
#print " client : %s startTime : %s endTime : %s" %(infos.clients[i], startTime, endTime )
if startTime >= endTime and logger != None :#to be removed
try:
logger.warning( _("Startime used in updateHourlyPickles was greater or equal to end time.") )
except:
pass
sp.pickleName = StatsPickler.buildThisHoursFileName( client = infos.clients[i], currentTime = startOfTheHour, machine = infos.machine, fileType = infos.fileType )
sp.collectStats( infos.types, startTime = startTime, endTime = endTime, interval = infos.interval * StatsDateLib.MINUTE, directory = pathToLogFiles, fileType = infos.fileType )
setLastUpdate( machine = infos.machine, client = infos.clients[i], fileType = infos.fileType, currentDate = infos.currentDate, paths = paths, collectUpToNow = infos.collectUpToNow )
def getOptionsFromParser( parser, logger = None ):
"""
This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the infos variable.
If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
( options, args )= parser.parse_args()
end = options.end.replace( '"','' ).replace( "'",'')
clients = options.clients.replace( ' ','' ).split( ',' )
machines = options.machines.replace( ' ','' ).split( ',' )
fileTypes = options.fileTypes.replace( ' ','' ).split( ',' )
products = options.products.replace( ' ','' ).split( ',' )
group = options.group.replace( ' ','' )
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( end, '%Y-%m-%d %H:%M:%S' )#will raise exception if format is wrong.
split = end.split()
currentTime = "%s %s" %( split[0], split[1] )
except:
print _( "Error. The endind date format must be YYYY-MM-DD HH:MM:SS" )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
#round ending hour to match pickleUpdater.
end = StatsDateLib.getIsoWithRoundedHours( end )
for machine in machines:
if machine != LOCAL_MACHINE:
GeneralStatsLibraryMethods.updateConfigurationFiles( machine, "pds" )
if products[0] != _("ALL") and group == "" :
print _( "Error. Products can only be specified when using special groups." )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
#init fileTypes array here if only one fileType is specified for all clients/sources
if len(fileTypes) == 1 and len(clients) !=1:
for i in range(1,len(clients) ):
fileTypes.append(fileTypes[0])
if clients[0] == _( "ALL" ) and fileTypes[0] != "":
print _( "Error. Filetypes cannot be specified when all clients are to be updated." )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
elif clients[0] != _( "ALL" ) and len(clients) != len( fileTypes ) :
print _( "Error. Specified filetypes must be either 1 for all the group or of the exact same lenght as the number of clients/sources." )
print _( "Use -h for help." )
print _( "Program terminated." )
sys.exit()
elif clients[0] == _( 'ALL' ) :
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machines[0] )
clients = []
clients.extend( txNames )
clients.extend( rxNames )
fileTypes = []
for txName in txNames:
fileTypes.append( _( "tx" ) )
for rxName in rxNames:
fileTypes.append( _( "rx" ) )
clients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters( end, 1000, clients, machines, fileTypes= fileTypes )
infos = _Infos( endTime = end, machines = machines, clients = clients, fileTypes = fileTypes, products = products, group = group )
return infos
def getOptionsFromParser( parser ):
"""
@summary : This method parses the argv received when
the program was called.
It takes the params wich have been passed by
the user and sets them in the corresponding
fields of the infos variable.
@note : If errors are encountered in parameters used,
it will immediatly terminate the application.
"""
currentTime = []
( options, args )= parser.parse_args()
collectUpToNow = options.collectUpToNow
timespan = options.timespan
machines = options.machines.replace( ' ','').split(',')
clientNames = options.clients.replace( ' ','' ).split(',')
types = options.types.replace( ' ', '').split(',')
currentTime = options.currentTime.replace('"','').replace("'",'')
fileType = options.fileType.replace("'",'')
collectUpToNow = options.collectUpToNow
copy = options.copy
combineClients = options.combineClients
productTypes = options.productTypes.replace( ' ', '' ).split( ',' )
groupName = options.groupName.replace( ' ','' )
outputLanguage = options.outputLanguage
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( currentTime, '%Y-%m-%d %H:%M:%S' )
split = currentTime.split()
currentTime = "%s %s" %( split[0], split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS" )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int( timespan ) < 1 :
raise
except:
print _("Error. The timespan value needs to be an integer one above 0." )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if groupName != "" and combineClients == False :
print _("Error. -g|--groupeName option requires the --combineClients options.")
print _('Group names are otherwise irrelevant.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
try :
if fileType == "tx":
validTypes = [ _("errors"), _("filecount"), _("bytecount"), _("latency") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
else:
validTypes = [ _("errors"), _("filecount"), _("bytecount") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
except:
print _("Error. With %s fileType, possible data types values are : %s.") %( fileType,validTypes )
print _('For multiple types use this syntax : -t "type1,type2"')
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. %s is not one of the supproted languages")
print _("Use one of the following languages : %s" % str( LanguageTools.getSupportedLanguages() ).replace("[","").replace("]","") )
print _("Use -h for additional help.")
print _("Program terminated.")
clientNames = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters( currentTime, timespan, clientNames, machines, [fileType])
directory = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, machines[0] )
infos = _GraphicsInfos( collectUpToNow = collectUpToNow, currentTime = currentTime, clientNames = clientNames,\
groupName = groupName, directory = directory , types = types, fileType = fileType, \
timespan = timespan, productTypes = productTypes, machines = machines, copy = copy, \
combineClients = combineClients, outputLanguage = outputLanguage )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours( infos.currentTime )
return infos
def getGraphicProducerFromParserOptions( parser ):
"""
This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the infos variable.
If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
graphicType = _("other")
mergerType = ""
( options, args )= parser.parse_args()
timespan = options.timespan
machines = options.machines.replace( ' ','').split(',')
clientNames = options.clients.replace( ' ','' ).split(',')
types = options.types.replace( ' ', '').split(',')
date = options.date.replace('"','').replace("'",'')
fileType = options.fileType.replace("'",'')
havingRun = options.havingRun
individual = options.individual
totals = options.totals
daily = options.daily
weekly = options.weekly
monthly = options.monthly
yearly = options.yearly
fixedCurrent = options.fixedCurrent
fixedPrevious = options.fixedPrevious
copy = options.copy
turnOffLogging = options.turnOffLogging
outputLanguage = options.outputLanguage
if outputLanguage == "":
outputLanguage = LanguageTools.getMainApplicationLanguage()
else :
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. The specified language is not currently supported by this application.")
print _("Please specify one of the following languages %s or use the default value()" %( str( LanguageTools.getSupportedLanguages() ).replace("[","").replace("]",""), LanguageTools.getMainApplicationLanguage() ) )
print _("Program terminated.")
sys.exit()
counter = 0
specialParameters = [daily, monthly, weekly, yearly]
for specialParameter in specialParameters:
if specialParameter:
counter = counter + 1
if counter > 1 :
print _("Error. Only one of the daily, weekly and yearly options can be use at a time ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif counter == 1 and timespan != None :
print _("Error. When using the daily, the weekly or the yearly options timespan cannot be specified. " )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif counter == 0:
if fixedPrevious or fixedCurrent:
print _("Error. When using one of the fixed options, please use either the -d -m -w or -y options. " )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if copy :
if daily or not( weekly or monthly or yearly ):
print _("Error. Copying can only be used with the -m -w or -y options. ")
print _("Use -h for help.")
print _("Program terminated.")
if counter == 0 and timespan == None :
timespan = 12
if fixedPrevious and fixedCurrent:
print _("Error. Please use only one of the fixed options,either fixedPrevious or fixedCurrent. ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if individual and totals:
print _("Error. Please use only one of the group options,either individual or totals. ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( date, '%Y-%m-%d %H:%M:%S' )
split = date.split()
date = "%s %s" %( split[0], split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
#Set graphic type based on parameters. Only one tpye is allowed at once based on previous validation.
if daily :
graphicType = _("daily")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24
elif weekly:
graphicType = _("weekly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 7
elif monthly:
graphicType = _("monthly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 30
elif yearly:
graphicType = _("yearly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 365
start, end = StatsDateLib.getStartEndInIsoFormat(date, timespan, graphicType, fixedCurrent, fixedPrevious )
timespan = int( StatsDateLib.getSecondsSinceEpoch( end ) - StatsDateLib.getSecondsSinceEpoch( start ) ) / 3600
#print "timespan %s" %timespan
try:
if int( timespan ) < 1 :
raise
except:
print _("Error. The timespan value needs to be an integer one above 0.")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print "Specified file type was : ", fileType
print _("Multiple types are not accepted.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if havingRun == True and clientNames[0] != _("ALL"):
print _("Error. Cannot use the havingRun option while specifying client/source names.")
print _("To use havingRun, do not use -c|--client option.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if clientNames[0] == _("ALL"):
# Get all of the client/sources that have run between graph's start and end.
if totals == True or havingRun == True :
#print start, end, machines
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( start, end, machines,None, havingrunOnAllMachines = True )
mergerType = _("totalForMachine")
else:#Build graphs only for currently runningclient/sources.
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machines[0] )
mergerType = _("group")
if fileType == _("tx"):
clientNames = txNames
#print clientNames
else:
clientNames = rxNames
else:
if totals == True :
mergerType = _("regular")
#--------------------------------------------------------------------- try :
if fileType == _("tx"):
validTypes = [ _("latency"), _("bytecount"), _("errors"), _("filesOverMaxLatency"), _("filecount") ]
if types[0] == _("All") :
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("%s is not a valid type" %t)
else:
validTypes = [ _("bytecount"), _("errors"), _("filecount") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
#------------------------------------------------------------------- except:
#----------------------------------------------------------- print types
# print _("Error. With %s fileType, possible data types values are : %s.") %( fileType, validTypes )
#---- print _("For multiple types use this syntax : -t 'type1','type2'")
#-------------------------------- print _("Use -h for additional help.")
#---------------------------------------- print _("Program terminated.")
#------------------------------------------------------------ sys.exit()
if individual != True :
combinedMachineName = ""
for machine in machines:
combinedMachineName = combinedMachineName + machine
machines = [ combinedMachineName ]
if len(clientNames) <1:
print _("Error. No client/sources were found that matched the specified parameters") %( fileType, validTypes )
print _("Verify parameters used, especially the machines parameter.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if len(clientNames) <1:
print _("Error. No client/sources were found that matched the specified parameters")
print _("Verify parameters used, especially the machines parameter.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
elif len(clientNames) == 1 and totals == True:
print _("Error. Cannot use totals option with only one client/source name.")
print _("Either remove --total option or use more than one client/source..")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
end = StatsDateLib.getIsoWithRoundedHours( end )
graphicsProducer = RRDGraphicProducer( startTime = start, endTime = end, graphicType = graphicType, clientNames = clientNames, types = types, timespan = timespan, machines = machines, fileType = fileType,\
totals = totals, copy = copy, mergerType = mergerType,turnOffLogging = turnOffLogging, inputLanguage = LanguageTools.getMainApplicationLanguage() , outputLanguage = outputLanguage )
return graphicsProducer
def main():
"""
@summary : Generates the web page based on the received
machines and file type parameters.
"""
try:
newForm = {}
form = cgi.FieldStorage()
for key in form.keys():
value = form.getvalue(key, "")
if isinstance(value, list):
# Multiple username fields specified
newvalue = ",".join(value)
else:
newvalue = value
newForm[key.replace("?","")]= newvalue
form = newForm
try:
language = form["lang"]
except:
language = ""
error = "Error.Unknwown language detected in popSourlientUpAdder"
try:
fileType = form['fileType']
if fileType !='tx' and fileType != 'rx':
error= "Error. File type needs to be either rx or tx."
except:
fileType = ""
try:
machines = form['machines']
machine = machines.split( ',' )[0]
machines = machines.replace( ',', '' )
except:
error = "Error. Machine names need to be specified."
machine = ""
if machines != "":
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine )
groups = getGroups( fileType, machine )
#print "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language )
#----------------------------- fileHandle = open("/web/pxStats/out","w")
# fileHandle.write(sys.path[0] + "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language ))
#---------------------------------------------------- fileHandle.close()
path = StatsPaths()
path.setPaths( language )
if fileType == "tx":
generateWebPage(txNames, groups, fileType, path.STATSWEBPAGESHTML + "/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language ), language )
elif fileType == "rx":
generateWebPage(rxNames, groups, fileType, path.STATSWEBPAGESHTML + "/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language ), language )
returnReply('')
except Exception, instance:
fileHandle = open("/web/pxStats/out","w")
fileHandle.write( "bug in main method" )
fileHandle.write( str( instance ) )
fileHandle.close()
sys.exit()
def main():
"""
@summary : Generates the web page based on the received
machines and file type parameters.
"""
try:
newForm = {}
form = cgi.FieldStorage()
for key in form.keys():
value = form.getvalue(key, "")
if isinstance(value, list):
# Multiple username fields specified
newvalue = ",".join(value)
else:
newvalue = value
newForm[key.replace("?", "")] = newvalue
form = newForm
try:
language = form["lang"]
except:
language = ""
error = "Error.Unknwown language detected in popSourlientUpAdder"
try:
fileType = form['fileType']
if fileType != 'tx' and fileType != 'rx':
error = "Error. File type needs to be either rx or tx."
except:
fileType = ""
try:
machines = form['machines']
machine = machines.split(',')[0]
machines = machines.replace(',', '')
except:
error = "Error. Machine names need to be specified."
machine = ""
if machines != "":
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, machine)
groups = getGroups(fileType, machine)
#print "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language )
#----------------------------- fileHandle = open("/web/pxStats/out","w")
# fileHandle.write(sys.path[0] + "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language ))
#---------------------------------------------------- fileHandle.close()
path = StatsPaths()
path.setPaths(language)
if fileType == "tx":
generateWebPage(
txNames, groups, fileType, path.STATSWEBPAGESHTML +
"/popUps/%s%sPopUpSourlientAdder_%s.html" %
(fileType, machines, language), language)
elif fileType == "rx":
generateWebPage(
rxNames, groups, fileType, path.STATSWEBPAGESHTML +
"/popUps/%s%sPopUpSourlientAdder_%s.html" %
(fileType, machines, language), language)
returnReply('')
except Exception, instance:
fileHandle = open("/web/pxStats/out", "w")
fileHandle.write("bug in main method")
fileHandle.write(str(instance))
fileHandle.close()
sys.exit()
def main():
"""
@summary : Gets all the parameters from config file.
Updates pickle files.
Generates all the required graphics.
Generates therwuired csv files.
Updates the different web pages.
Updates the desired databases.
Uploads graphics to the required machines.
Monitors the result of all the activities.
"""
if GeneralStatsLibraryMethods.processIsAlreadyRunning( "pxStatsStartup" ) == False:
setGlobalLanguageParameters()
GeneralStatsLibraryMethods.createLockFile( "pxStatsStartup" )
currentTime = time.time()
currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch( currentTime )
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
validateParameters( generalParameters, machineParameters, None )
tagsNeedingUpdates = getMachinesTagsNeedingUpdates( generalParameters, machineParameters )
if tagsNeedingUpdates == None : #no previous parameter found
saveCurrentMachineParameters( machineParameters )
elif tagsNeedingUpdates != [] :
updateFilesAssociatedWithMachineTags( tagsNeedingUpdates, machineParameters )
saveCurrentMachineParameters( machineParameters )
updatePickles( generalParameters, machineParameters, currentTimeInIsoFormat )
updateDatabases( generalParameters, machineParameters, currentTimeInIsoFormat )
backupRRDDatabases( generalParameters, currentTimeInIsoFormat, generalParameters.nbDbBackupsToKeep )
#updateCsvFiles( )
getGraphicsForWebPages( generalParameters, currentTimeInIsoFormat )
updateWebPages( generalParameters )
#uploadGraphicFiles( generalParameters, machineParameters )
cleanUp( generalParameters , currentTimeInIsoFormat, generalParameters.daysOfPicklesToKeep )
monitorActivities( generalParameters, currentTimeInIsoFormat )
updateManager = AutomaticUpdatesManager( generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup" )
updateManager.addAutomaticUpdateToLogs( currentTimeInIsoFormat )
GeneralStatsLibraryMethods.deleteLockFile( "pxStatsStartup" )
print _( "Finished." )
else:
print _( "Error. An other instance of pxStatsStartup is allready running." )
print _( "Only one instance of this software can be run at once." )
print _( "Please terminate the other instance or wait for it to end it's execution" )
print _( "before running this program again." )
print _( "Program terminated." )
sys.exit()
def generateGraphsForPairedMachines(infos):
"""
@summary : Create graphs for all client
by merging the data from all
the listed machines.
"""
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, infos.machines[0])
#print infos.machines
#print txNames
infos.combinedName = str(infos.machines).replace(' ', '').replace(
'[', '').replace(']', '')
j = 0
for txName in txNames:
pid = os.fork() #create child process
if pid == 0: #child process
output = commands.getoutput(
"python %sgenerateGnuGraphics.py -m %s -f tx -c %s -d '%s' -s %s --copy --outputLanguage %s"
% (STATSPATHS.STATSBIN, infos.combinedName, txName, infos.date,
infos.timespan, infos.outputLanguage))
print output
sys.exit() #terminate child process
else:
#print "wait"
j = j + 1
if j % 10 == 0:
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
j = 0
for rxName in rxNames:
pid = os.fork() #create child process
if pid == 0: #child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m %s -f rx -c %s -d '%s' -s %s --copy --outputLanguage %s"\
%( STATSPATHS.STATSBIN, infos.combinedName, rxName, infos.date, infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j % 10 == 0:
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
#print "goes to wait"
pid, status = os.wait()
except:
break
def getParametersFromForm( self, form ):
"""
@summary: Initialises the queryParameters
based on the form received as
parameter.
@note : Absent parameters will be set to default values( [] or '' )
and will NOT raise exceptions. Use the searchForParameterErrors
function to search for errors
"""
global _
#print form
image = None #No image was produced yet
#Every param is received in an array, use [0] to get first item, nothing for array.
try:
querier = form["querier"].replace("'", "").replace('"','')
except:
querier = ''
try:
plotter = form["plotter"].replace("'", "").replace('"','')
except:
plotter = ''
try:
fileTypes = form["fileType"].replace("'", "").replace('"','')
except:
fileTypes = ''
try:
sourLients = form["sourLients"].split(',')
except:
sourLients = []
try:
groupName = form["groupName"].replace("'", "").replace('"','')
except:
groupName = ''
try:
machines = form["machines"].split(',')
except:
machines = []
if groupName != '' and ( sourLients == [] or sourLients == [''] ) :
configParameters = StatsConfigParameters( )
configParameters.getAllParameters()
if groupName in configParameters.groupParameters.groups:
if configParameters.groupParameters.groupFileTypes[groupName] == fileTypes and configParameters.groupParameters.groupsMachines[groupName] == machines:
sourLients = configParameters.groupParameters.groupsMembers[groupName]
try:
combine = form["combineSourlients"].replace(",", "").replace('"','')
if combine == 'false' or combine == 'False':
combine = False
elif combine == 'true' or combine == 'True':
combine = True
else:
raise
except:
combine = False
try:
endTime = form["endTime"].replace("'", "").replace('"','')
hour = endTime.split(" ")[1]
splitDate = endTime.split(" ")[0].split( '-' )
endTime = "%s" %( splitDate[2] + '-' + splitDate[1] + '-' + splitDate[0] + " " + hour )
if _("current") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromCurrentDay(endTime)
elif _("previous") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromPreviousDay( endTime )
except:
endTime = ''
try:
products = form["products"].split(',')
if products == [""]:
raise
except:
products = ["*"]
try:
statsTypes = form["statsTypes"].split(',')
except:
statsTypes = []
#statsTypes = translateStatsTypes( statsTypes )
try:
span = form["span"].replace("'", "").replace('"','')
if str(span).replace( ' ', '' ) == '':
raise
span = int(span)
except:
span = 24
try:
language = form["lang"].replace("'", "").replace('"','')
except:
language = ""
sourLients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters( endTime, span, sourLients, machines, [fileTypes])
self.queryParameters = GnuQueryBroker._QueryParameters( fileTypes, sourLients, groupName, machines, combine, endTime, products, statsTypes, span, language )
self.replyParameters = GnuQueryBroker._ReplyParameters( querier, plotter, image, fileTypes, sourLients, groupName, machines, combine, endTime, products, statsTypes,
span, '', language )
def getParametersFromForm(self, form):
"""
@summary: Initialises the queryParameters
based on the form received as
parameter.
@note : Absent parameters will be set to default values( [] or '' )
and will NOT raise exceptions. Use the searchForParameterErrors
function to search for errors
"""
global _
#print form
image = None #No image was produced yet
#Every param is received in an array, use [0] to get first item, nothing for array.
try:
querier = form["querier"].replace("'", "").replace('"', '')
except:
querier = ''
try:
plotter = form["plotter"].replace("'", "").replace('"', '')
except:
plotter = ''
try:
fileTypes = form["fileType"].replace("'", "").replace('"', '')
except:
fileTypes = ''
try:
sourLients = form["sourLients"].split(',')
except:
sourLients = []
try:
groupName = form["groupName"].replace("'", "").replace('"', '')
except:
groupName = ''
try:
machines = form["machines"].split(',')
except:
machines = []
if groupName != '' and (sourLients == [] or sourLients == ['']):
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
if groupName in configParameters.groupParameters.groups:
if configParameters.groupParameters.groupFileTypes[
groupName] == fileTypes and configParameters.groupParameters.groupsMachines[
groupName] == machines:
sourLients = configParameters.groupParameters.groupsMembers[
groupName]
try:
combine = form["combineSourlients"].replace(",",
"").replace('"', '')
if combine == 'false' or combine == 'False':
combine = False
elif combine == 'true' or combine == 'True':
combine = True
else:
raise
except:
combine = False
try:
endTime = form["endTime"].replace("'", "").replace('"', '')
hour = endTime.split(" ")[1]
splitDate = endTime.split(" ")[0].split('-')
endTime = "%s" % (splitDate[2] + '-' + splitDate[1] + '-' +
splitDate[0] + " " + hour)
if _("current") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromCurrentDay(
endTime)
elif _("previous") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromPreviousDay(
endTime)
except:
endTime = ''
try:
products = form["products"].split(',')
if products == [""]:
raise
except:
products = ["*"]
try:
statsTypes = form["statsTypes"].split(',')
except:
statsTypes = []
#statsTypes = translateStatsTypes( statsTypes )
try:
span = form["span"].replace("'", "").replace('"', '')
if str(span).replace(' ', '') == '':
raise
span = int(span)
except:
span = 24
try:
language = form["lang"].replace("'", "").replace('"', '')
except:
language = ""
sourLients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
endTime, span, sourLients, machines, [fileTypes])
self.queryParameters = GnuQueryBroker._QueryParameters(
fileTypes, sourLients, groupName, machines, combine, endTime,
products, statsTypes, span, language)
self.replyParameters = GnuQueryBroker._ReplyParameters(
querier, plotter, image, fileTypes, sourLients, groupName,
machines, combine, endTime, products, statsTypes, span, '',
language)
def getOptionsFromParser( parser, paths, logger = None ):
"""
@summary : This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the hlE variable.
@Warning : If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
directories = []
startTimes = []
( options, args ) = parser.parse_args()
interval = options.interval
collectUpToNow = options.collectUpToNow
currentDate = options.currentDate.replace( '"','' ).replace( "'",'' )
currentDate = StatsDateLib.getIsoWithRoundedHours( currentDate )
fileType = options.fileType.replace( "'",'' )
machine = options.machine.replace( " ","" )
clients = options.clients.replace(' ','' ).split( ',' )
types = options.types.replace( ' ', '' ).split( ',' )
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles( LOCAL_MACHINE, machine )
#print "*****pathToLogFiles %s" %pathToLogFiles
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( currentDate, '%Y-%m-%d %H:%M:%S' )
split = currentDate.split()
currentDate = "%s %s" %( split[0],split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS" )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try:
if int( interval ) < 1 :
raise
except:
print _("Error. The interval value needs to be an integer one above 0." )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print _('Multiple types are not accepted.' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if fileType == "tx":
validTypes = [ _("errors"), _("latency"), _("bytecount") ]
else:
validTypes = [ _("errors"), _("bytecount") ]
if types[0] == _("All"):
types = validTypes
try :
for t in types :
if t not in validTypes:
raise
except:
print _("Error. With %s fileType, possible data types values are : %s.") %(fileType,validTypes )
print _('For multiple types use this syntax : -t "type1,type2"' )
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
def translateType(typeToTranslate):
translations = { _("errors"):"errors", _("latency"):"latency", _("bytecount"):"bytecount" }
return translations[typeToTranslate]
types = map( translateType, types )
if clients[0] == _("All") :
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine )
if fileType == "tx":
clients = txNames
else:
clients = rxNames
#print "clients found :%s" %clients
# Verify that each client needs to be updated.
# If not we add a warning to the logger and removwe the client from the list
# since it's not needed, but other clients might be.
usefullClients = []
for client in clients :
startTime = getLastUpdate( machine = machine, client = client, fileType= fileType, currentDate = currentDate , paths = paths, collectUpToNow = collectUpToNow )
if currentDate > startTime:
#print " client : %s currentDate : %s startTime : %s" %( client, currentDate, startTime )
directories.append( pathToLogFiles )
startTimes.append( startTime )
usefullClients.append( client )
else:
#print "This client was not updated since it's last update was more recent than specified date : %s" %client
if logger != None :
try:
logger.warning( _("This client was not updated since it's last update was more recent than specified date : %s") %client)
except :
pass
infos = _UpdaterInfos( currentDate = currentDate, clients = usefullClients, startTimes = startTimes, directories = directories ,\
types = types, collectUpToNow = collectUpToNow, fileType = fileType, machine = machine )
if collectUpToNow == False:
infos.endTime = StatsDateLib.getIsoWithRoundedHours( infos.currentDate )
return infos
usefull = self.containsUsefullInfo(fileName)
if usefull == True:
self.entries.append(fileName)
else:
if self.logger != None:
self.logger.warning(_("Warning in LogFileCollector. Folder named %s does not exist.") % self.directory)
if __name__ == "__main__":
"""
Small test case. Tests if everything works plus gives an idea on proper usage.
"""
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles(LOCAL_MACHINE, LOCAL_MACHINE)
lfc = LogFileCollector(
startTime="2006-07-20 01:00:00",
endTime="2006-07-20 02:00:00",
directory=pathToLogFiles,
lastLineRead="",
logType="tx",
name="satnet",
)
lfc.collectEntries()
print "Files returned : %s " % lfc.entries
def main():
"""
@summary : Gets all the parameters from config file.
Updates pickle files.
Generates all the required graphics.
Generates therwuired csv files.
Updates the different web pages.
Updates the desired databases.
Uploads graphics to the required machines.
Monitors the result of all the activities.
"""
if GeneralStatsLibraryMethods.processIsAlreadyRunning(
"pxStatsStartup") == False:
setGlobalLanguageParameters()
GeneralStatsLibraryMethods.createLockFile("pxStatsStartup")
currentTime = time.time()
currentTimeInIsoFormat = StatsDateLib.getIsoFromEpoch(currentTime)
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
validateParameters(generalParameters, machineParameters, None)
tagsNeedingUpdates = getMachinesTagsNeedingUpdates(
generalParameters, machineParameters)
if tagsNeedingUpdates == None: #no previous parameter found
saveCurrentMachineParameters(machineParameters)
elif tagsNeedingUpdates != []:
updateFilesAssociatedWithMachineTags(tagsNeedingUpdates,
machineParameters)
saveCurrentMachineParameters(machineParameters)
updatePickles(generalParameters, machineParameters,
currentTimeInIsoFormat)
updateDatabases(generalParameters, machineParameters,
currentTimeInIsoFormat)
backupRRDDatabases(generalParameters, currentTimeInIsoFormat,
generalParameters.nbDbBackupsToKeep)
#updateCsvFiles( )
getGraphicsForWebPages(generalParameters, currentTimeInIsoFormat)
updateWebPages(generalParameters)
#uploadGraphicFiles( generalParameters, machineParameters )
cleanUp(generalParameters, currentTimeInIsoFormat,
generalParameters.daysOfPicklesToKeep)
monitorActivities(generalParameters, currentTimeInIsoFormat)
updateManager = AutomaticUpdatesManager(
generalParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
updateManager.addAutomaticUpdateToLogs(currentTimeInIsoFormat)
GeneralStatsLibraryMethods.deleteLockFile("pxStatsStartup")
print _("Finished.")
else:
print _(
"Error. An other instance of pxStatsStartup is allready running.")
print _("Only one instance of this software can be run at once.")
print _(
"Please terminate the other instance or wait for it to end it's execution"
)
print _("before running this program again.")
print _("Program terminated.")
sys.exit()
def generateGraphsForPairedMachines( infos ) :
"""
@summary : Create graphs for all client
by merging the data from all
the listed machines.
"""
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, infos.machines[0] )
#print infos.machines
#print txNames
infos.combinedName = str(infos.machines).replace( ' ','' ).replace( '[','' ).replace( ']', '' )
j=0
for txName in txNames :
pid = os.fork()#create child process
if pid == 0 :#child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m %s -f tx -c %s -d '%s' -s %s --copy --outputLanguage %s"
%( STATSPATHS.STATSBIN, infos.combinedName, txName, infos.date, infos.timespan, infos.outputLanguage ) )
print output
sys.exit() #terminate child process
else:
#print "wait"
j = j + 1
if j %10 == 0:
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
j=0
for rxName in rxNames:
pid = os.fork()#create child process
if pid == 0:#child process
output = commands.getoutput( "python %sgenerateGnuGraphics.py -m %s -f rx -c %s -d '%s' -s %s --copy --outputLanguage %s"\
%( STATSPATHS.STATSBIN, infos.combinedName, rxName, infos.date, infos.timespan, infos.outputLanguage ) )
print output
sys.exit()
else:
j = j + 1
if j %10 == 0:
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
#print "goes to wait"
pid, status = os.wait( )
except:
break
def getMaximumGaps( self ):
"""
@summary : Reads columbos
maxSettings.conf file
@returns: The maximum gaps dictionnary containing the
maximium gap allowed between two transfers
for a specific client. ex : { clientX: 60 }
"""
allNames = []
rxNames = []
txNames = []
maximumGaps = {}
try:
machineConfig = MachineConfigParameters()
machineConfig.getParametersFromMachineConfigurationFile()
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
self.__updateMaxSettingsFile( generalParameters )
for tag in generalParameters.sourceMachinesTags:
try:
#print tag
#print "in getMaximumGaps %s" %generalParameters.detailedParameters.sourceMachinesForTag
machines = generalParameters.detailedParameters.sourceMachinesForTag[tag]
machine = machines[0]
except:
raise Exception( "Invalid tag found in main configuration file." )
newRxNames, newTxNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine )
rxNames.extend(newRxNames)
txNames.extend(newTxNames)
allNames.extend( rxNames )
allNames.extend( txNames )
circuitsRegex, default_circuit, timersRegex, default_timer, pxGraphsRegex, default_pxGraph = readMaxFile.readQueueMax( self.maxSettingsFile, "PX" )
for key in timersRegex.keys(): #fill all explicitly set maximum gaps.
values = timersRegex[key]
newKey = key.replace( "^", "" ).replace( "$","").replace(".","")
maximumGaps[newKey] = values
for name in allNames:#add all clients/sources for wich no value was set
if name not in maximumGaps.keys(): #no value was set
nameFoundWithWildcard = False
for key in timersRegex.keys(): # in case a wildcard character was used
cleanKey = key.replace( "^", "" ).replace( "$","").replace(".","")
if fnmatch.fnmatch( name, cleanKey ):
maximumGaps[name] = timersRegex[key]
nameFoundWithWildcard = True
if nameFoundWithWildcard == False :
maximumGaps[name] = default_timer
except:#in case of corrupt or absent file
maximumGaps = {}
return maximumGaps
if usefull == True:
self.entries.append(fileName)
else:
if self.logger != None:
self.logger.warning(
_("Warning in LogFileCollector. Folder named %s does not exist."
) % self.directory)
if __name__ == "__main__":
"""
Small test case. Tests if everything works plus gives an idea on proper usage.
"""
pathToLogFiles = GeneralStatsLibraryMethods.getPathToLogFiles(
LOCAL_MACHINE, LOCAL_MACHINE)
lfc = LogFileCollector(startTime="2006-07-20 01:00:00",
endTime="2006-07-20 02:00:00",
directory=pathToLogFiles,
lastLineRead="",
logType="tx",
name="satnet")
lfc.collectEntries()
print "Files returned : %s " % lfc.entries
def getOptionsFromParser(parser, logger=None):
"""
This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the infos variable.
If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
(options, args) = parser.parse_args()
end = options.end.replace('"', '').replace("'", '')
clients = options.clients.replace(' ', '').split(',')
machines = options.machines.replace(' ', '').split(',')
fileTypes = options.fileTypes.replace(' ', '').split(',')
products = options.products.replace(' ', '').split(',')
group = options.group.replace(' ', '')
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime(
end,
'%Y-%m-%d %H:%M:%S') #will raise exception if format is wrong.
split = end.split()
currentTime = "%s %s" % (split[0], split[1])
except:
print _("Error. The endind date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
#round ending hour to match pickleUpdater.
end = StatsDateLib.getIsoWithRoundedHours(end)
for machine in machines:
if machine != LOCAL_MACHINE:
GeneralStatsLibraryMethods.updateConfigurationFiles(machine, "pds")
if products[0] != _("ALL") and group == "":
print _(
"Error. Products can only be specified when using special groups.")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
#init fileTypes array here if only one fileType is specified for all clients/sources
if len(fileTypes) == 1 and len(clients) != 1:
for i in range(1, len(clients)):
fileTypes.append(fileTypes[0])
if clients[0] == _("ALL") and fileTypes[0] != "":
print _(
"Error. Filetypes cannot be specified when all clients are to be updated."
)
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif clients[0] != _("ALL") and len(clients) != len(fileTypes):
print _(
"Error. Specified filetypes must be either 1 for all the group or of the exact same lenght as the number of clients/sources."
)
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif clients[0] == _('ALL'):
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames(
LOCAL_MACHINE, machines[0])
clients = []
clients.extend(txNames)
clients.extend(rxNames)
fileTypes = []
for txName in txNames:
fileTypes.append(_("tx"))
for rxName in rxNames:
fileTypes.append(_("rx"))
clients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
end, 1000, clients, machines, fileTypes=fileTypes)
infos = _Infos(endTime=end,
machines=machines,
clients=clients,
fileTypes=fileTypes,
products=products,
group=group)
return infos