def getTimeParametersFromConfigurationFile(self):
"""
@summary : Reads all the time related parameters of the
config file and sets them in the timeParameters
attribute's values.
"""
self.timeParameters = TimeConfigParameters()
self.timeParameters.getTimeParametersFromConfigurationFile()
def getTimeParametersFromConfigurationFile(self):
"""
@summary : Reads all the time related parameters of the
config file and sets them in the timeParameters
attribute's values.
"""
self.timeParameters = TimeConfigParameters()
self.timeParameters.getTimeParametersFromConfigurationFile()
def getCurrentUpdateFrequency(self):
"""
@summary : Returns the current update frequency
in minutes base on the crontab entries.
@return :
"""
currentUpdateFrequency = 0
if self.updateType == "pxStatsStartup" :
self.__getPxStatsCurrentUpdateFrequency()
else:
timeParameters = TimeConfigParameters()
timeParameters.getTimeParametersFromConfigurationFile()
frequencyFromConfig= {}
if self.updateType == "monitoring":
frequencyFromConfig = timeParameters.monitoringFrequency
elif self.updateType == "generalCleaner":
frequencyFromConfig = timeParameters.generalCleanerFrequency
elif self.updateType == "picklecleaner":
frequencyFromConfig = timeParameters.pickleCleanerFrequency
elif self.updateType == "dbBackups":
frequencyFromConfig = timeParameters.dbBackupsFrequency
else:
raise Exception( "Unsupported updateType detected in AutomaticUpdatesManager. Supported types are : %s" %( SUPPORTED_UPDATE_TYPES ) )
value = frequencyFromConfig.keys()[0]
if frequencyFromConfig[value] == "minutes":
currentUpdateFrequency = int(value)
elif frequencyFromConfig[value] == "hours":
currentUpdateFrequency = int(value) * 60
elif frequencyFromConfig[value] == "days":
currentUpdateFrequency = int(value) * 60 * 24
return currentUpdateFrequency
class StatsConfigParameters:
"""
This class is usefull to store all the values
collected from the config file into a single
object.
"""
def __init__( self, sourceMachinesTags = None, picklingMachines = None , machinesToBackupInDb = None , \
graphicsUpLoadMachines = None, daysOfPicklesToKeep = None, nbDbBackupsToKeep = None, \
nbAutoUpdatesLogsToKeep = None, timeParameters = None, detailedParameters = None, \
groupParameters = None, mainApplicationLanguage = 'en',\
artifactsLanguages = None, webPagesLanguages = None, statsRoot = '/apps/px/pxStats/' ):
"""
@param sourceMachinesTags : Name tags used to depict the machine
containing the data source.
@param picklingMachines : Machines where the pickling occurs.
@param machinesToBackupInDb : Names of the machines whose data needs
to be backed up in databases.
@param graphicsUpLoadMachines : Machines to which graphics needs to be uploaded to.
@param daysOfPickledtoKeep : Number of days worth of backups we need to preserve.
@param nbDbBackupsToKeep : Number of database backups to keep. Span will be
determined by this number and the frequecy of
the backups.
@param nbAutoUpdatesLogsToKeep : Number of automatic updates log files to keep.
@param timeParameters : TimeConfigParameters() instance.
@param detailedParameters : DetailedStatsParameters() instance.
@param groupParameters : GroupConfigParameters instance.
@param mainApplicationLanguage : Language to use throughout the application.
@param webPagesLanguages : List containing the ( displayedLanguage, fileLanguage )
tuples detailing how the web pages are to be generated.
@param statsRoot : Root path of the stats library.
"""
self.sourceMachinesTags = sourceMachinesTags or []
self.picklingMachines = picklingMachines or []
self.machinesToBackupInDb = machinesToBackupInDb or []
self.graphicsUpLoadMachines = graphicsUpLoadMachines or []
self.daysOfPickledtoKeep = daysOfPicklesToKeep
self.nbDbBackupsToKeep = nbDbBackupsToKeep
self.nbAutoUpdatesLogsToKeep = nbAutoUpdatesLogsToKeep
self.timeParameters = timeParameters
self.detailedParameters = detailedParameters
self.groupParameters = groupParameters
self.mainApplicationLanguage = mainApplicationLanguage
self.artifactsLanguages = artifactsLanguages or []
self.webPagesLanguages = webPagesLanguages or []
self.statsRoot = statsRoot
def getDetailedParametersFromMachineConfig( self ):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map( None, self.sourceMachinesTags, self.picklingMachines ):
sourceMachines = machineParameters.getMachinesAssociatedWith( machineTag )
picklingMachines = machineParameters.getMachinesAssociatedWith( picklingMachine )
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] =[]
for machine in sourceMachines :
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(machine)
self.detailedParameters.sourceMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[machineTag]:
self.detailedParameters.picklingMachines[machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith( uploadMachine )
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[machine] = machineParameters.getUserNameForMachine(machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines !=[]:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(machine)
def getGeneralParametersFromStatsConfigurationFile(self):
"""
@summary : Gathers GENERAL parameters from the
StatsPath.STATSETC/config file.
@note : Does not set groupParameters, time parameters
and detailed parameters.
@return : None
"""
paths = StatsPaths()
paths.setBasicPaths()
configFile = paths.STATSETC + "config"
config = ConfigParser()
file = open( configFile )
config.readfp( file )
self.sourceMachinesTags = []
self.picklingMachines = []
self.machinesToBackupInDb = []
self.graphicsUpLoadMachines = []
self.artifactsLanguages = []
self.webPagesLanguages = []
self.mainApplicationLanguage = config.get( 'generalConfig', 'mainApplicationLanguage' )
self.artifactsLanguages.extend( config.get( 'generalConfig', 'artifactsLanguages' ).split(',') )
languagePairs = config.get( 'generalConfig', 'webPagesLanguages' ).split(',')
for languagePair in languagePairs:
self.webPagesLanguages.append( (languagePair.split(":")[0], languagePair.split(":")[1]) )
self.statsRoot = config.get( 'generalConfig', 'statsRoot' )
self.sourceMachinesTags.extend( config.get( 'generalConfig', 'sourceMachinesTags' ).split(',') )
self.picklingMachines.extend( config.get( 'generalConfig', 'picklingMachines' ).split(',') )
self.machinesToBackupInDb.extend( config.get( 'generalConfig', 'machinesToBackupInDb' ).split(',') )
self.graphicsUpLoadMachines.extend( config.get( 'generalConfig', 'graphicsUpLoadMachines' ).split(',') )
self.daysOfPicklesToKeep = float( config.get( 'generalConfig', 'daysOfPicklesToKeep' ) )
self.nbDbBackupsToKeep = float( config.get( 'generalConfig', 'nbDbBackupsToKeep' ) )
self.nbAutoUpdatesLogsToKeep = int( config.get( 'generalConfig', 'nbAutoUpdatesLogsToKeep' ) )
try:
file.close()
except:
pass
def getTimeParametersFromConfigurationFile(self):
"""
@summary : Reads all the time related parameters of the
config file and sets them in the timeParameters
attribute's values.
"""
self.timeParameters = TimeConfigParameters()
self.timeParameters.getTimeParametersFromConfigurationFile()
def getGroupSettingsFromConfigurationFile( self ):
"""
Reads all the group settings from
the configuration file.
"""
groupParameters = GroupConfigParameters([], {}, {}, {},{} )
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
paths = StatsPaths()
paths.setBasicPaths()
config = paths.STATSETC + "config"
fileHandle = open( config, "r" )
line = fileHandle.readline()#read until groups section, or EOF
while line != "" and "[specialGroups]" not in line:
line = fileHandle.readline()
if line != "":#read until next section, or EOF
line = fileHandle.readline()
while line != "" and "[" not in line:
if line != '\n' and line[0] != '#' :
splitLine = line.split()
if len( splitLine ) == 6:
groupName = splitLine[0]
if groupName not in (groupParameters.groups):
groupParameters.groups.append( groupName )
groupParameters.groupsMachines[groupName] = []
groupParameters.groupFileTypes[groupName] = []
groupParameters.groupsMembers[groupName] = []
groupParameters.groupsProducts[groupName] = []
machines = splitLine[2].split(",")
for machine in machines:
groupParameters.groupsMachines[groupName].extend( machineParameters.getMachinesAssociatedWith(machine) )
groupParameters.groupFileTypes[groupName] = splitLine[3]
groupParameters.groupsMembers[groupName].extend( splitLine[4].split(",") )
groupParameters.groupsProducts[groupName].extend( splitLine[5].split(",") )
line = fileHandle.readline()
fileHandle.close()
self.groupParameters = groupParameters
def getAllParameters(self):
"""
@summary : Get all the parameters form the different configuration files.
"""
self.getGeneralParametersFromStatsConfigurationFile()
self.getGroupSettingsFromConfigurationFile()
self.getDetailedParametersFromMachineConfig()
self.getTimeParametersFromConfigurationFile()
class StatsConfigParameters:
"""
This class is usefull to store all the values
collected from the config file into a single
object.
"""
def __init__( self, sourceMachinesTags = None, picklingMachines = None , machinesToBackupInDb = None , \
graphicsUpLoadMachines = None, daysOfPicklesToKeep = None, nbDbBackupsToKeep = None, \
nbAutoUpdatesLogsToKeep = None, timeParameters = None, detailedParameters = None, \
groupParameters = None, mainApplicationLanguage = 'en',\
artifactsLanguages = None, webPagesLanguages = None, statsRoot = '/apps/px/pxStats/' ):
"""
@param sourceMachinesTags : Name tags used to depict the machine
containing the data source.
@param picklingMachines : Machines where the pickling occurs.
@param machinesToBackupInDb : Names of the machines whose data needs
to be backed up in databases.
@param graphicsUpLoadMachines : Machines to which graphics needs to be uploaded to.
@param daysOfPickledtoKeep : Number of days worth of backups we need to preserve.
@param nbDbBackupsToKeep : Number of database backups to keep. Span will be
determined by this number and the frequecy of
the backups.
@param nbAutoUpdatesLogsToKeep : Number of automatic updates log files to keep.
@param timeParameters : TimeConfigParameters() instance.
@param detailedParameters : DetailedStatsParameters() instance.
@param groupParameters : GroupConfigParameters instance.
@param mainApplicationLanguage : Language to use throughout the application.
@param webPagesLanguages : List containing the ( displayedLanguage, fileLanguage )
tuples detailing how the web pages are to be generated.
@param statsRoot : Root path of the stats library.
"""
self.sourceMachinesTags = sourceMachinesTags or []
self.picklingMachines = picklingMachines or []
self.machinesToBackupInDb = machinesToBackupInDb or []
self.graphicsUpLoadMachines = graphicsUpLoadMachines or []
self.daysOfPickledtoKeep = daysOfPicklesToKeep
self.nbDbBackupsToKeep = nbDbBackupsToKeep
self.nbAutoUpdatesLogsToKeep = nbAutoUpdatesLogsToKeep
self.timeParameters = timeParameters
self.detailedParameters = detailedParameters
self.groupParameters = groupParameters
self.mainApplicationLanguage = mainApplicationLanguage
self.artifactsLanguages = artifactsLanguages or []
self.webPagesLanguages = webPagesLanguages or []
self.statsRoot = statsRoot
def getDetailedParametersFromMachineConfig(self):
'''
@summary: Sets all the detailed parameters found in the
config files based on what is read in the
config file and the machine config file.
@note: All parameters for this object should be set prior
to calling this method.
'''
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
self.detailedParameters = DetailedStatsParameters()
for machineTag, picklingMachine in map(None, self.sourceMachinesTags,
self.picklingMachines):
sourceMachines = machineParameters.getMachinesAssociatedWith(
machineTag)
picklingMachines = machineParameters.getMachinesAssociatedWith(
picklingMachine)
if sourceMachines != []:
self.detailedParameters.sourceMachinesForTag[machineTag] = []
self.detailedParameters.picklingMachines[machineTag] = []
for machine in sourceMachines:
if machine not in self.detailedParameters.individualSourceMachines:
self.detailedParameters.sourceMachinesForTag[
machineTag].append(machine)
self.detailedParameters.individualSourceMachines.append(
machine)
self.detailedParameters.sourceMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for machine in picklingMachines:
if machine not in self.detailedParameters.sourceMachinesForTag[
machineTag]:
self.detailedParameters.picklingMachines[
machineTag].append(machine)
self.detailedParameters.picklingMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for uploadMachine in self.graphicsUpLoadMachines:
uploadMachines = machineParameters.getMachinesAssociatedWith(
uploadMachine)
if uploadMachines != []:
for machine in uploadMachines:
if machine not in self.detailedParameters.uploadMachines:
self.detailedParameters.uploadMachines.append(machine)
self.detailedParameters.uploadMachinesLogins[
machine] = machineParameters.getUserNameForMachine(
machine)
for dbMachine in self.machinesToBackupInDb:
dbMachines = machineParameters.getMachinesAssociatedWith(dbMachine)
if dbMachines != []:
for machine in dbMachines:
if machine not in self.detailedParameters.databaseMachines:
self.detailedParameters.databaseMachines.append(
machine)
def getGeneralParametersFromStatsConfigurationFile(self):
"""
@summary : Gathers GENERAL parameters from the
StatsPath.STATSETC/config file.
@note : Does not set groupParameters, time parameters
and detailed parameters.
@return : None
"""
paths = StatsPaths()
paths.setBasicPaths()
configFile = paths.STATSETC + "config"
config = ConfigParser()
file = open(configFile)
config.readfp(file)
self.sourceMachinesTags = []
self.picklingMachines = []
self.machinesToBackupInDb = []
self.graphicsUpLoadMachines = []
self.artifactsLanguages = []
self.webPagesLanguages = []
self.mainApplicationLanguage = config.get('generalConfig',
'mainApplicationLanguage')
self.artifactsLanguages.extend(
config.get('generalConfig', 'artifactsLanguages').split(','))
languagePairs = config.get('generalConfig',
'webPagesLanguages').split(',')
for languagePair in languagePairs:
self.webPagesLanguages.append(
(languagePair.split(":")[0], languagePair.split(":")[1]))
self.statsRoot = config.get('generalConfig', 'statsRoot')
self.sourceMachinesTags.extend(
config.get('generalConfig', 'sourceMachinesTags').split(','))
self.picklingMachines.extend(
config.get('generalConfig', 'picklingMachines').split(','))
self.machinesToBackupInDb.extend(
config.get('generalConfig', 'machinesToBackupInDb').split(','))
self.graphicsUpLoadMachines.extend(
config.get('generalConfig', 'graphicsUpLoadMachines').split(','))
self.daysOfPicklesToKeep = float(
config.get('generalConfig', 'daysOfPicklesToKeep'))
self.nbDbBackupsToKeep = float(
config.get('generalConfig', 'nbDbBackupsToKeep'))
self.nbAutoUpdatesLogsToKeep = int(
config.get('generalConfig', 'nbAutoUpdatesLogsToKeep'))
try:
file.close()
except:
pass
def getTimeParametersFromConfigurationFile(self):
"""
@summary : Reads all the time related parameters of the
config file and sets them in the timeParameters
attribute's values.
"""
self.timeParameters = TimeConfigParameters()
self.timeParameters.getTimeParametersFromConfigurationFile()
def getGroupSettingsFromConfigurationFile(self):
"""
Reads all the group settings from
the configuration file.
"""
groupParameters = GroupConfigParameters([], {}, {}, {}, {})
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
paths = StatsPaths()
paths.setBasicPaths()
config = paths.STATSETC + "config"
fileHandle = open(config, "r")
line = fileHandle.readline() #read until groups section, or EOF
while line != "" and "[specialGroups]" not in line:
line = fileHandle.readline()
if line != "": #read until next section, or EOF
line = fileHandle.readline()
while line != "" and "[" not in line:
if line != '\n' and line[0] != '#':
splitLine = line.split()
if len(splitLine) == 6:
groupName = splitLine[0]
if groupName not in (groupParameters.groups):
groupParameters.groups.append(groupName)
groupParameters.groupsMachines[groupName] = []
groupParameters.groupFileTypes[groupName] = []
groupParameters.groupsMembers[groupName] = []
groupParameters.groupsProducts[groupName] = []
machines = splitLine[2].split(",")
for machine in machines:
groupParameters.groupsMachines[
groupName].extend(
machineParameters.
getMachinesAssociatedWith(machine))
groupParameters.groupFileTypes[
groupName] = splitLine[3]
groupParameters.groupsMembers[groupName].extend(
splitLine[4].split(","))
groupParameters.groupsProducts[groupName].extend(
splitLine[5].split(","))
line = fileHandle.readline()
fileHandle.close()
self.groupParameters = groupParameters
def getAllParameters(self):
"""
@summary : Get all the parameters form the different configuration files.
"""
self.getGeneralParametersFromStatsConfigurationFile()
self.getGroupSettingsFromConfigurationFile()
self.getDetailedParametersFromMachineConfig()
self.getTimeParametersFromConfigurationFile()