def cleanUp( configParameters, currentTime, daysOfPicklesToKeep ): """ @summary: Based on current time and frequencies contained within the time parameters, we will run the cleaners that need to be run. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner") if updateManager.updateIsRequired(currentTime) : output = commands.getoutput( paths.STATSTOOLS + "pickleCleaner.py %s" %int(daysOfPicklesToKeep) ) print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep ) updateManager.addAutomaticUpdateToLogs( currentTime ) updateManager = AutomaticUpdatesManager(configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner") if updateManager.updateIsRequired(currentTime) : commands.getstatusoutput( paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" ) print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" updateManager.addAutomaticUpdateToLogs( currentTime )
def getPreviousMonitoringJob( self, currentTime ): """ @summary : Gets the previous crontab from the pickle file. @return : Time of the previous monitoring job. @warning : Returns "" if file does not exist. """ statsPaths = StatsPaths() statsPaths.setPaths() file = "%spreviousMonitoringJob" %statsPaths.STATSMONITORING previousMonitoringJob = "" if os.path.isfile( file ): fileHandle = open( file, "r" ) previousMonitoringJob = pickle.load( fileHandle ) fileHandle.close() #print previousMonitoringJob else: previousMonitoringJob = StatsDateLib.getIsoTodaysMidnight( currentTime ) #print previousMonitoringJob return previousMonitoringJob
def __updateCsvFiles( self, type, clusters, cost ): """ @summary : Generate th rx and tx csv files for yesterday for all clusters. @param type : daily | weekly | monthly | yearly @param clusters : List of currently running source clusters. @param cost : total operational cost for the perido specified by the type @return : None """ paths = StatsPaths() paths.setPaths() typeParameters = { "daily" : "-d", "weekly" : "-w", "monthly" : "-m", "yearly" : "-y" } output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) ) #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) ) #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) fileName = self.__getFileNameFromExecutionOutput(output) if fileName != "": commands.getstatusoutput(paths.STATSWEBPAGESGENERATORS + 'csvDataFiltersForWebPages.py -c %s -f %s ' %(cost, fileName) )
def addAutomaticUpdateToLogs( self, timeOfUpdateInIsoFormat, currentUpdateFrequency = None ): """ @summary : Writes a new file in the log folder containing the current update frequency. @param timeOfUpdateInIsoFormat: Time that the entries name will sport. """ paths = StatsPaths() paths.setPaths() fileName = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + str( timeOfUpdateInIsoFormat ).replace( " ", "_" ) #Safety to make sure if not os.path.isdir( os.path.dirname( fileName ) ): os.makedirs( os.path.dirname( fileName ), 0777 ) if currentUpdateFrequency == None : currentUpdateFrequency = self.getCurrentUpdateFrequency() CpickleWrapper.save( currentUpdateFrequency, fileName ) allEntries = os.listdir(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/") allEntries.sort() entriesToRemove = allEntries[ :-self.numberOfLogsToKeep] for entrytoRemove in entriesToRemove: os.remove(paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" + entrytoRemove )
def saveList( self, user, clients ): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir( directory ): os.makedirs( directory, mode=0777 ) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save( object = self.savedFileList, filename = completeFilename )
def __getDocFilesToLinkTo(self, language): """ @summary : Gathers and returns all the documentation files currently available @summary : The list of fileNames to link to. """ filesToLinkTo = [] statsPaths = StatsPaths() statsPaths.setPaths( self.mainLanguage ) folder = statsPaths.STATSDOC + "html/" listOfFilesInFolder = os.listdir(folder) for file in listOfFilesInFolder: baseName = os.path.basename(file) if( fnmatch.fnmatch( baseName, "*_%s.html"%(language) ) ): filesToLinkTo.append( baseName ) filesToLinkTo.sort() return filesToLinkTo
def backupRRDDatabases(configParameters, currentTime, nbBackupsToKeep): """ @summary: Based on current time and frequencies contained within the time parameters, we will backup the databases only if necessary. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "dbBackups") if updateManager.updateIsRequired(currentTime): commands.getstatusoutput(paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str(int(nbBackupsToKeep))) print paths.STATSTOOLS + "backupRRDDatabases.py" + " " + str( nbBackupsToKeep) updateManager.addAutomaticUpdateToLogs(currentTime)
def cleanUp(configParameters, currentTime, daysOfPicklesToKeep): """ @summary: Based on current time and frequencies contained within the time parameters, we will run the cleaners that need to be run. @param configParameters: StatsConfigParameters instance. @param currenTime: currentTime in seconds since epoch format. """ paths = StatsPaths() paths.setPaths() updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "picklecleaner") if updateManager.updateIsRequired(currentTime): output = commands.getoutput(paths.STATSTOOLS + "pickleCleaner.py %s" % int(daysOfPicklesToKeep)) print paths.STATSTOOLS + "pickleCleaner.py" + " " + str( daysOfPicklesToKeep) updateManager.addAutomaticUpdateToLogs(currentTime) updateManager = AutomaticUpdatesManager( configParameters.nbAutoUpdatesLogsToKeep, "generalCleaner") if updateManager.updateIsRequired(currentTime): commands.getstatusoutput(paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf") print paths.STATSTOOLS + "clean_dir.plx" + " " + paths.STATSETC + "clean_dir.conf" updateManager.addAutomaticUpdateToLogs(currentTime)
def getClientsCurrentFileList( self, clients ): """ @summary : Gets all the files associated with the list of clients. @note : Client list is used here since we need to find all the pickles that will be used in a merger. Thus unlike all other methods we dont refer here to the combined name but rather to a list of individual machine names. @summary : Returns the all the files in a dictionnary associated with each file associated with it's mtime. """ fileNames = [] statsPaths = StatsPaths() statsPaths.setPaths() for client in clients : filePattern = statsPaths.STATSPICKLES + client + "/*/*" #_?? folderNames = glob.glob( filePattern ) for folder in folderNames: if os.path.isdir( folder ): filePattern = folder + "/" + "*_??" fileNames.extend( glob.glob( filePattern ) ) for fileName in fileNames : self.currentClientFileList[fileName] = os.path.getmtime( fileName ) return self.currentClientFileList
def updateFilesAssociatedWithMachineTags(tagsNeedingUpdates, machineParameters): """ @summary : For all the tags for wich a machine was change we rename all the files associated with that tag. @param tagsNeedingUpdates: List of tags that have been modified since the last call. """ paths = StatsPaths() paths.setPaths() previousParameters = getMachineParametersFromPreviousCall() for tag in tagsNeedingUpdates: previousCombinedMachineNames = "" previousCombinedMachineNames = previousCombinedMachineNames.join( [x for x in previousParameters.getMachinesAssociatedWith(tag)]) currentCombinedMachineNames = "" currentCombinedMachineNames = currentCombinedMachineNames.join( [x for x in machineParameters.getMachinesAssociatedWith(tag)]) output = commands.getoutput( "%sfileRenamer.py -o %s -n %s --overrideConfirmation" % (paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames)) print "%sfileRenamer.py -o %s -n %s --overrideConfirmation" % ( paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames) print output
def restoreDatabaseUpdateTimes(timeToRestore, currentTime, nbBackupsToKeep): """ @summary : Copy all databases into a folder sporting the data of the backup. @param timeToRestore : Time of the DB backups to set as current DB. @param currentTime : Time of the call to the script. @param nbBackupsToKeep : total number of backups to keep. """ statsPaths = StatsPaths() statsPaths.setPaths() source = statsPaths.STATSDBUPDATESBACKUPS + "/%s" % timeToRestore destination = statsPaths.STATSCURRENTDBUPDATES #Archive current Database backupRRDDatabases.backupDatabaseUpdateTimes(currentTime, nbBackupsToKeep, foldersToPreserve=[source]) #restore desired status, output = commands.getstatusoutput("rm -r %s" % (destination)) os.makedirs(destination) status, output = commands.getstatusoutput("cp -rf %s/* %s" % (source, destination)) print output
def getTimeOfLastUpdateInLogs(self): """ @summary : Returns the time of the last update in iso format. @return : None if no update as found, EPCH is returned in iso format, as to make sure an update is made since no prior updates exist. """ timeOfLastUpdate = StatsDateLib.getIsoTodaysMidnight( StatsDateLib.getCurrentTimeInIsoformat() ) paths = StatsPaths() paths.setPaths() updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" if not os.path.isdir( updatesDirectory ): os.makedirs(updatesDirectory) allEntries = os.listdir(updatesDirectory) if allEntries !=[] : allEntries.sort() allEntries.reverse() timeOfLastUpdate = os.path.basename( allEntries[0] ).replace( "_"," " ) return timeOfLastUpdate
def updateWordsFromDB(wordType, word, language): """ @summary: Updates words within the db depending on the specified type ofdatabases @param wordType : Type of word : "products" or "groupName" @parameter language : Language that is currently used by the caller. @param word : Word to add to the database @return : None """ _ = LanguageTools.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, language) statsPaths = StatsPaths() statsPaths.setPaths(language) if wordType == "products": updateWordsFromFile(statsPaths.STATSWEBWORDDATABASES + _('products'), word) elif wordType == "groupName": updateWordsFromFile(statsPaths.STATSWEBWORDDATABASES + _('groupNames'), word)
def __getAutomaticUpdatesDoneDuringTimeSpan( self, startTime, endtime ): """ @param startTime: Start time of the span in iso format @param endtime: end time of the span in iso format """ #set to fit file standard startTime = startTime.replace( " ", "_" ) endtime = endtime.replace( " ", "_" ) def afterEndTime(x): return x <= endtime def beforeStartTime(x): return x >= startTime paths = StatsPaths() paths.setPaths() updates = os.listdir( updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/" ) updates = filter( afterEndTime, updates) updates = filter( beforeStartTime, updates) return updates
def __getDocFilesToLinkTo(self, language): """ @summary : Gathers and returns all the documentation files currently available @summary : The list of fileNames to link to. """ filesToLinkTo = [] statsPaths = StatsPaths() statsPaths.setPaths(self.mainLanguage) folder = statsPaths.STATSDOC + "html/" listOfFilesInFolder = os.listdir(folder) for file in listOfFilesInFolder: baseName = os.path.basename(file) if (fnmatch.fnmatch(baseName, "*_%s.html" % (language))): filesToLinkTo.append(baseName) filesToLinkTo.sort() return filesToLinkTo
def restoreDatabaseUpdateTimes( timeToRestore, currentTime, nbBackupsToKeep ): """ @summary : Copy all databases into a folder sporting the data of the backup. @param timeToRestore : Time of the DB backups to set as current DB. @param currentTime : Time of the call to the script. @param nbBackupsToKeep : total number of backups to keep. """ statsPaths = StatsPaths() statsPaths.setPaths() source = statsPaths.STATSDBUPDATESBACKUPS + "/%s" %timeToRestore destination = statsPaths.STATSCURRENTDBUPDATES #Archive current Database backupRRDDatabases.backupDatabaseUpdateTimes( currentTime, nbBackupsToKeep, foldersToPreserve = [ source ] ) #restore desired status, output = commands.getstatusoutput( "rm -r %s" %( destination ) ) os.makedirs(destination) status, output = commands.getstatusoutput( "cp -rf %s/* %s" %( source, destination ) ) print output
def updatePickledTimes( dateToSet = "2006-10-23 09:00:00" ): """ @summary : Get all the keys then set all of them to the desired date. """ statsPaths = StatsPaths() statsPaths.setPaths() folder = statsPaths.STATSPICKLESTIMEOFUPDATES files = os.listdir(folder) for fileName in files : if os.path.isfile( fileName ): fileHandle = open( fileName, "r" ) pickledTimes = pickle.load( fileHandle ) fileHandle.close() keys = pickledTimes.keys() for key in keys: pickledTimes[key] = dateToSet fileHandle = open( fileName, "w" ) pickle.dump( pickledTimes, fileHandle ) fileHandle.close()
def updateFilesAssociatedWithMachineTags( tagsNeedingUpdates, machineParameters ): """ @summary : For all the tags for wich a machine was change we rename all the files associated with that tag. @param tagsNeedingUpdates: List of tags that have been modified since the last call. """ paths = StatsPaths() paths.setPaths() previousParameters = getMachineParametersFromPreviousCall() for tag in tagsNeedingUpdates: previousCombinedMachineNames = "" previousCombinedMachineNames = previousCombinedMachineNames.join( [ x for x in previousParameters.getMachinesAssociatedWith( tag ) ] ) currentCombinedMachineNames = "" currentCombinedMachineNames = currentCombinedMachineNames.join( [ x for x in machineParameters.getMachinesAssociatedWith( tag ) ]) output = commands.getoutput( "%sfileRenamer.py -o %s -n %s --overrideConfirmation" %( paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames ) ) print "%sfileRenamer.py -o %s -n %s --overrideConfirmation" %( paths.STATSTOOLS, previousCombinedMachineNames, currentCombinedMachineNames ) print output
def getSavedList(self, user, clients): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try: self.savedFileList = CpickleWrapper.load(directory + fileName) if self.savedFileLis == None: self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def updatePickledTimes(dateToSet="2006-10-23 09:00:00"): """ @summary : Get all the keys then set all of them to the desired date. """ statsPaths = StatsPaths() statsPaths.setPaths() folder = statsPaths.STATSPICKLESTIMEOFUPDATES files = os.listdir(folder) for fileName in files: if os.path.isfile(fileName): fileHandle = open(fileName, "r") pickledTimes = pickle.load(fileHandle) fileHandle.close() keys = pickledTimes.keys() for key in keys: pickledTimes[key] = dateToSet fileHandle = open(fileName, "w") pickle.dump(pickledTimes, fileHandle) fileHandle.close()
def getSavedList( self, user, clients ): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try : self.savedFileList = CpickleWrapper.load( directory + fileName ) if self.savedFileLis == None : self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def main(): """ @summary : Gathers options, then makes call to transferPickleToRRD """ paths = StatsPaths() paths.setPaths() language = 'en' setGlobalLanguageParameters() createPaths(paths) logger = Logger(paths.STATSLOGGING + 'stats_' + 'rrd_transfer' + '.log.notb', 'INFO', 'TX' + 'rrd_transfer', bytes=10000000) logger = logger.getLogger() parser = createParser() infos = getOptionsFromParser(parser, logger=logger) transferPickleToRRD(infos, logger=logger)
def saveList(self, user, clients): """ @summary : Saves list. @note : Will include modification made in updateFileInlist method @param clients : Client to wich the file is related(used to narrow down searchs) @param user : Name of the client, person, etc.. wich has a relation with the file. """ statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user if not os.path.isdir(directory): os.makedirs(directory, mode=0777) #create directory completeFilename = directory + fileName #print "saving %s" %completeFilename CpickleWrapper.save(object=self.savedFileList, filename=completeFilename)
def getClientsCurrentFileList(self, clients): """ @summary : Gets all the files associated with the list of clients. @note : Client list is used here since we need to find all the pickles that will be used in a merger. Thus unlike all other methods we dont refer here to the combined name but rather to a list of individual machine names. @summary : Returns the all the files in a dictionnary associated with each file associated with it's mtime. """ fileNames = [] statsPaths = StatsPaths() statsPaths.setPaths() for client in clients: filePattern = statsPaths.STATSPICKLES + client + "/*/*" #_?? folderNames = glob.glob(filePattern) for folder in folderNames: if os.path.isdir(folder): filePattern = folder + "/" + "*_??" fileNames.extend(glob.glob(filePattern)) for fileName in fileNames: self.currentClientFileList[fileName] = os.path.getmtime( fileName) return self.currentClientFileList
def transferLogFiles(): """ @summary : Log files will not be tansferred if local machine is not designed to be a pickling machine. If log files are to be transferred, they will be straight from the source." """ paths = StatsPaths() paths.setPaths() parameters = StatsConfigParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() parameters.getAllParameters() individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags ) individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines ) for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) : if picklingMachine == LOCAL_MACHINE :#pickling to be done here userName = machineParameters.getUserNameForMachine(sourceMachine) remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName ) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine ) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) ) print output
def updateCsvFiles(): """ @summary : Runs the csv file update utility. """ paths = StatsPaths() paths.setPaths() output = commands.getoutput( "%sgetCsvFilesforWebPages.py" %paths.STATSWEBPAGESGENERATORS )
def __init__( self, directory, fileType, clientNames = None , groupName = "", timespan = 12,\ currentTime = None, productTypes = None, logger = None, logging = True, machines = None,\ workingLanguage = None, outputLanguage = None ): """ @summary : ClientGraphicProducer constructor. CurrentTime format is ISO meaning "2006-06-8 00:00:00". Will use current system time by default. CurrentTime is to be used if a different time than sytem time is to be used. Very usefull for testing or to implement graphic request where user can choose start time. """ global _ _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, workingLanguage) if currentTime != None: currentTime = currentTime else: currentTime = time.time() self.directory = directory # Directory where log files are located. self.fileType = fileType # Type of log files to be used. self.machines = machines or [] # Machines for wich to collect data. self.clientNames = clientNames or [ ] # Client name we need to get the data from. self.groupName = groupName # Name for a group of clients to be combined. self.timespan = timespan # Number of hours we want to gather the data from. self.currentTime = currentTime # Time when stats were queried. self.productTypes = productTypes or [ "All" ] # Specific data types on wich we'll collect the data. self.loggerName = 'graphs' # Name of the logger self.logger = logger # Logger to use is logging == true. self.logging = logging # Whether or not to enable logging. self.outputLanguage = outputLanguage # Language in which the graphic will be produced in. paths = StatsPaths() paths.setPaths() if logging == True: if self.logger is None: # Enable logging if not os.path.isdir(paths.STATSLOGGING): os.makedirs(paths.STATSLOGGING, mode=0777) self.logger = Logger( paths.STATSLOGGING + 'stats_' + self.loggerName + '.log.notb', 'INFO',\ 'TX' + self.loggerName, bytes = 10000000 ) self.logger = self.logger.getLogger() else: self.logger = None
def updateCsvFiles(): """ @summary : Runs the csv file update utility. """ paths = StatsPaths() paths.setPaths() output = commands.getoutput("%sgetCsvFilesforWebPages.py" % paths.STATSWEBPAGESGENERATORS)
def giveOutPermissionsToFolders(currentlyUsedLanguages): """ @summary : opens up permissions to folders that might be required by the web user. @param currentlyUsedLanguages: Languages currently set to be displayed in the web interface """ for language in currentlyUsedLanguages: _ = LanguageTools.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, language) paths = StatsPaths() paths.setPaths(language) pathsToOpenUp = [] pathsToOpenUp.append(paths.STATSLOGGING) pathsToOpenUp.append(paths.STATSPICKLES) pathsToOpenUp.append(paths.STATSDB) pathsToOpenUp.append(paths.STATSCURRENTDB) pathsToOpenUp.append(paths.STATSCURRENTDB + _("bytecount")) pathsToOpenUp.append(paths.STATSCURRENTDB + _("errors")) pathsToOpenUp.append(paths.STATSCURRENTDB + _("filecount")) pathsToOpenUp.append(paths.STATSCURRENTDB + _("filesOverMaxLatency")) pathsToOpenUp.append(paths.STATSCURRENTDB + _("latency")) pathsToOpenUp.append(paths.STATSCURRENTDBUPDATES) pathsToOpenUp.append(paths.STATSCURRENTDBUPDATES + _("rx")) pathsToOpenUp.append(paths.STATSCURRENTDBUPDATES + _("tx")) pathsToOpenUp.append(paths.STATSCURRENTDBUPDATES + _("totals")) pathsToOpenUp.append(paths.STATSDBBACKUPS) pathsToOpenUp.append(paths.STATSDBBACKUPS + "*/" + _("rx")) pathsToOpenUp.append(paths.STATSDBBACKUPS + "*/" + _("tx")) pathsToOpenUp.append(paths.STATSDBBACKUPS + "*/" + _("totals")) pathsToOpenUp.append(paths.STATSGRAPHS) pathsToOpenUp.append(paths.STATSGRAPHS + _("others/")) pathsToOpenUp.append(paths.STATSGRAPHS + _("others/") + "gnuplot/") pathsToOpenUp.append(paths.STATSGRAPHS + _("others/") + "rrd/") pathsToOpenUp.append(paths.STATSWEBPAGESHTML + "/popUps/") for path in pathsToOpenUp: if not os.path.isdir(path): os.makedirs(path, 0777) commands.getstatusoutput("chmod 0777 %s" % path) commands.getstatusoutput("chmod 0777 %s/*" % path)
def transfer(login, machine): """ @summary : Transfers all the required files from specified machine into local machine. Every task is done 10 times. This is done in hope that everything downloaded will be as coherent as possible. If a large amount of file is to be transferred, some files that were downloaded at the beginning of the transfer,wich can take up to a few hours,might not be as up to date as the file downloaded towards the end of the transfer """ statsPaths = StatsPaths() statsPaths.setPaths() localPaths = getLocalPathsToSynchronise() buildLocalPaths(localPaths) configFiles = getConfigFiles() for i in range(10): transferLogFiles() for localPath in localPaths: remotePath = statsPaths.getStatsPathFromMachine( localPath, machine, login) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s" % ( login, machine, remotePath, localPath) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s" % (login, machine, remotePath, localPath)) print output for configfile in configFiles: #get real path localPath = os.path.dirname(configfile) fileName = os.path.basename(configfile) remotePath = statsPaths.getStatsPathFromMachine( localPath, machine, login) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s" % ( login, machine, remotePath + fileName, localPath + fileName) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s" % (login, machine, remotePath + fileName, localPath + fileName)) print output
def giveOutPermissionsToFolders( currentlyUsedLanguages ): """ @summary : opens up permissions to folders that might be required by the web user. @param currentlyUsedLanguages: Languages currently set to be displayed in the web interface """ for language in currentlyUsedLanguages: _ = LanguageTools.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, language) paths = StatsPaths() paths.setPaths(language) pathsToOpenUp = [] pathsToOpenUp.append( paths.STATSLOGGING) pathsToOpenUp.append( paths.STATSPICKLES ) pathsToOpenUp.append( paths.STATSDB) pathsToOpenUp.append( paths.STATSCURRENTDB ) pathsToOpenUp.append( paths.STATSCURRENTDB + _("bytecount") ) pathsToOpenUp.append( paths.STATSCURRENTDB + _("errors") ) pathsToOpenUp.append( paths.STATSCURRENTDB + _("filecount") ) pathsToOpenUp.append( paths.STATSCURRENTDB + _("filesOverMaxLatency")) pathsToOpenUp.append( paths.STATSCURRENTDB + _("latency")) pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES) pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("rx") ) pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("tx") ) pathsToOpenUp.append( paths.STATSCURRENTDBUPDATES + _("totals") ) pathsToOpenUp.append( paths.STATSDBBACKUPS ) pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("rx") ) pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("tx") ) pathsToOpenUp.append( paths.STATSDBBACKUPS + "*/" + _("totals") ) pathsToOpenUp.append( paths.STATSGRAPHS ) pathsToOpenUp.append( paths.STATSGRAPHS +_("others/")) pathsToOpenUp.append( paths.STATSGRAPHS +_("others/") + "gnuplot/") pathsToOpenUp.append( paths.STATSGRAPHS +_("others/") + "rrd/") pathsToOpenUp.append( paths.STATSWEBPAGESHTML + "/popUps/") for path in pathsToOpenUp: if not os.path.isdir(path): os.makedirs(path, 0777) commands.getstatusoutput( "chmod 0777 %s" %path ) commands.getstatusoutput( "chmod 0777 %s/*" %path )
def buildCsvFileName(infos): """ @summary: Builds and returns the file name to use for the csv file. @param infos: _CvsInfos instance containing the required information to build up the file name. @return: Return the built up file name. """ global _ StatsDateLib.setLanguage(infos.outputLanguage) paths = StatsPaths() paths.setPaths(infos.outputLanguage) machinesStr = str(infos.machinesForLabels).replace('[', '').replace( ']', '').replace(',', '').replace("'", "").replace('"', '').replace(' ', '') currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime( StatsDateLib.getSecondsSinceEpoch(infos.start)) currentWeek = time.strftime( "%W", time.gmtime(StatsDateLib.getSecondsSinceEpoch(infos.start))) fileName = paths.STATSCSVFILES if infos.span == "daily": fileName = fileName + "/" + _( "daily/") + infos.fileType + "/%s/%s/%s/%s.csv" % ( machinesStr, currentYear, currentMonth, currentDay) elif infos.span == "weekly": fileName = fileName + "/" + _( "weekly/") + infos.fileType + "/%s/%s/%s.csv" % ( machinesStr, currentYear, currentWeek) elif infos.span == "monthly": fileName = fileName + "/" + _( "monthly/") + infos.fileType + "/%s/%s/%s.csv" % ( machinesStr, currentYear, currentMonth) elif infos.span == "yearly": fileName = fileName + "/" + _( "yearly/") + infos.fileType + "/%s/%s.csv" % (machinesStr, currentYear) StatsDateLib.setLanguage(LanguageTools.getMainApplicationLanguage()) return fileName
def main(): """ Parses parameters then calls the tranferMethod. """ setGlobalLanguageParameters() paths = StatsPaths() paths.setPaths() login = "" machine = "" if len( sys.argv ) == 3 : login = sys.argv[1] machine = sys.argv[2] transfer( login, machine ) else: print _( "#######################################################################################################" ) print _( "#" ) print _( "# Help for retreiveDataFromMachine.py" ) print _( "#" ) print _( "# This program is to be used to transfer all of the important stats files" ) print _( "# from a remote machine to the local machine." ) print _( "#" ) print _( "# If large transfers are to be done, program may take many hours to complete." ) print _( "# Output from every operation will be printed as to let the user see exactly what is going on" ) print _( "# and allow errors to be detected." ) print _( "#" ) print _( "# Examples : invalid login or ssh other ssh errors" ) print _( "#" ) print _( "# This will also serve as to take out the guesswork as to why the program is taking so long to complete." ) print _( "#" ) print _( "# Log files will not be tansferred if local machine is not designed to be a pickling machine. " ) print _( "# If log files are to be transferred, they will be transferred straight from the source." ) print _( "#" ) print _( "# *** Make sure %sconfig is filled properly prior to running this script !!! ***") %paths.STATSETC print _( "# *** Consult pxStats documentation if you are unsure how to fill %sconfig. ***") %paths.STATSETC print _( "#" ) print _( "#" ) print _( "#" ) print _( "# Usage : Program must receive exactly two arguments." ) print _( "#" ) print _( "# Example : python retreiveDataFromMachine.py login machineName " ) print _( "#" ) print _( "#######################################################################################################") print "" print "" print "" sys.exit()
def __init__( self, directory, fileType, clientNames = None , groupName = "", timespan = 12,\ currentTime = None, productTypes = None, logger = None, logging = True, machines = None,\ workingLanguage = None, outputLanguage = None ): """ @summary : ClientGraphicProducer constructor. CurrentTime format is ISO meaning "2006-06-8 00:00:00". Will use current system time by default. CurrentTime is to be used if a different time than sytem time is to be used. Very usefull for testing or to implement graphic request where user can choose start time. """ global _ _ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, workingLanguage ) if currentTime != None : currentTime = currentTime else: currentTime = time.time() self.directory = directory # Directory where log files are located. self.fileType = fileType # Type of log files to be used. self.machines = machines or [] # Machines for wich to collect data. self.clientNames = clientNames or [] # Client name we need to get the data from. self.groupName = groupName # Name for a group of clients to be combined. self.timespan = timespan # Number of hours we want to gather the data from. self.currentTime = currentTime # Time when stats were queried. self.productTypes = productTypes or ["All"] # Specific data types on wich we'll collect the data. self.loggerName = 'graphs' # Name of the logger self.logger = logger # Logger to use is logging == true. self.logging = logging # Whether or not to enable logging. self.outputLanguage = outputLanguage # Language in which the graphic will be produced in. paths = StatsPaths() paths.setPaths() if logging == True: if self.logger is None: # Enable logging if not os.path.isdir( paths.STATSLOGGING ): os.makedirs( paths.STATSLOGGING , mode=0777 ) self.logger = Logger( paths.STATSLOGGING + 'stats_' + self.loggerName + '.log.notb', 'INFO',\ 'TX' + self.loggerName, bytes = 10000000 ) self.logger = self.logger.getLogger() else: self.logger = None
def getLocalPathsToSynchronise(): """ @summary : Returns the list of local paths that are required to with the remote machine we want to be synchronised with. """ statsPaths = StatsPaths() statsPaths.setPaths() paths = [ statsPaths.STATSMONITORING , statsPaths.STATSGRAPHS , statsPaths.STATSPICKLES, statsPaths.STATSCURRENTDB,\ statsPaths.STATSDBBACKUPS , statsPaths.STATSCURRENTDBUPDATES, statsPaths.STATSDBUPDATESBACKUPS ] return paths
def uploadGraphicFiles( parameters, machineParameters ): """ @summary : Takes all the created daily graphics dedicated to clumbo and uploads them to the machines specified in the parameters. """ paths = StatsPaths() paths.setPaths() for uploadMachine in parameters.graphicsUpLoadMachines : output = commands.getoutput( "scp %s* %s@%s:%s " %( paths.STATSCOLGRAPHS, machineParameters.getUserNameForMachine(uploadMachine), uploadMachine, paths.PDSCOLGRAPHS ) ) print "scp %s* %s@%s:%s " %( paths.STATSCOLGRAPHS, machineParameters.getUserNameForMachine(uploadMachine),uploadMachine, paths.PDSCOLGRAPHS ) print output
def updateDatabases( parameters, machineParameters, currentTimeInIsoFormat ): """ @summary : Updates all the required databases by transferring the data found in the pickle files into rrd databases files. First transfers all the pickles into databases for all the clusters. Then combines all the data required by the different groups found within the config file. @param parameters: StatsConfigParameters instance containing the parameters found in the config file. @param machineParameters: MachineConfigParameters instance containing the parameters found in the config file. @param currentTimeInIsoFormat : Time at which this program was originally called. @return : None """ paths = StatsPaths() paths.setPaths() #Small safety measure in case another instance of the program is allready running. if transferToDatabaseAlreadyRunning() == False : for tag in parameters.machinesToBackupInDb : machines = machineParameters.getMachinesAssociatedWith(tag) machines = str( machines ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) output = commands.getoutput( "%stransferPickleToRRD.py -m '%s' -e '%s' " %( paths.STATSBIN, machines, currentTimeInIsoFormat ) ) print "%stransferPickleToRRD.py -m '%s' " %( paths.STATSBIN, machines ) print "output:%s" %output if parameters.groupParameters.groups != []: for group in parameters.groupParameters.groups : groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupMachines = str( parameters.groupParameters.groupsMachines[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupProducts = str( parameters.groupParameters.groupsProducts[group] ).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) groupFileTypes = str(parameters.groupParameters.groupFileTypes[group]).replace( "[", "" ).replace( "]", "" ).replace( " ", "" ) output = commands.getoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -g '%s' -f %s -p '%s' -e '%s' " %( paths.STATSBIN, groupMembers, groupMachines, group, groupFileTypes, groupProducts, currentTimeInIsoFormat ) ) print "%stransferPickleToRRD.py -c '%s' -m '%s' -g '%s' -f %s -p '%s' " %( paths.STATSBIN, groupMembers, groupMachines, group, groupFileTypes, groupProducts ) print output
def getListOfPickleUpdateFiles(): """ @summary: Returns the list of currently available pickle update files @return: Returns the list of currently available pickle update files """ statsPaths = StatsPaths() statsPaths.setPaths() files = glob.glob( statsPaths.STATSPICKLESTIMEOFUPDATES + '*' ) return files
def saveCurrentMachineParameters(machineParameters): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)): os.makedirs(os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS)) CpickleWrapper.save(machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def getListOfPickleUpdateFiles(): """ @summary: Returns the list of currently available pickle update files @return: Returns the list of currently available pickle update files """ statsPaths = StatsPaths() statsPaths.setPaths() files = glob.glob(statsPaths.STATSPICKLESTIMEOFUPDATES + "*") return files
def saveCurrentMachineParameters( machineParameters ): """ @summary : Saves the current machineParameters into the /data/previousMachineParameters file. @param machineParameters: Machine parameters to save. """ paths = StatsPaths() paths.setPaths() if not os.path.isdir( os.path.dirname( paths.STATSPREVIOUSMACHINEPARAMS ) ): os.makedirs( os.path.dirname(paths.STATSPREVIOUSMACHINEPARAMS) ) CpickleWrapper.save( machineParameters, paths.STATSPREVIOUSMACHINEPARAMS)
def __updateCsvFiles(self, type, clusters, cost): """ @summary : Generate th rx and tx csv files for yesterday for all clusters. @param type : daily | weekly | monthly | yearly @param clusters : List of currently running source clusters. @param cost : total operational cost for the perido specified by the type @return : None """ paths = StatsPaths() paths.setPaths() typeParameters = { "daily": "-d", "weekly": "-w", "monthly": "-m", "yearly": "-y" } output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s' % (typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage)) #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f rx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) output = commands.getoutput( paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s' % (typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage)) #print paths.STATSBIN + 'csvDataConversion.py --includeGroups %s --machines "%s" --machinesAreClusters --fixedPrevious --date "%s" -f tx --language %s' %( typeParameters[type], clusters, self.timeOfRequest, self.outputLanguage ) fileName = self.__getFileNameFromExecutionOutput(output) if fileName != "": commands.getstatusoutput( paths.STATSWEBPAGESGENERATORS + 'csvDataFiltersForWebPages.py -c %s -f %s ' % (cost, fileName))
def transfer( login, machine ): """ @summary : Transfers all the required files from specified machine into local machine. Every task is done 10 times. This is done in hope that everything downloaded will be as coherent as possible. If a large amount of file is to be transferred, some files that were downloaded at the beginning of the transfer,wich can take up to a few hours,might not be as up to date as the file downloaded towards the end of the transfer """ statsPaths = StatsPaths() statsPaths.setPaths() localPaths = getLocalPathsToSynchronise() buildLocalPaths( localPaths ) configFiles = getConfigFiles() for i in range( 10 ): transferLogFiles() for localPath in localPaths : remotePath = statsPaths.getStatsPathFromMachine( localPath, machine, login ) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s" %( login, machine, remotePath, localPath ) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s" %( login, machine, remotePath, localPath ) ) print output for configfile in configFiles: #get real path localPath = os.path.dirname( configfile ) fileName = os.path.basename( configfile ) remotePath = statsPaths.getStatsPathFromMachine( localPath, machine, login ) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s" %( login, machine, remotePath + fileName, localPath + fileName ) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s" %( login, machine, remotePath + fileName, localPath + fileName ) ) print output
def getConfigFiles() : """ @summary : Returns the list of config files to synchronise. @return : Returns the list of config files to synchronise. """ statsPaths = StatsPaths() statsPaths.setPaths() configFiles = [] configFiles.append( statsPaths.STATSMONITORING + 'maxSettings.conf' ) configFiles.append( statsPaths.STATSMONITORING + 'previousCrontab' ) configFiles.append( statsPaths.STATSMONITORING + 'previousFileChecksums') return configFiles
def getMachineParametersFromPreviousCall() : """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile( paths.STATSPREVIOUSMACHINEPARAMS ): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS ) return previousMachineParams
def getConfigFiles(): """ @summary : Returns the list of config files to synchronise. @return : Returns the list of config files to synchronise. """ statsPaths = StatsPaths() statsPaths.setPaths() configFiles = [] configFiles.append(statsPaths.STATSMONITORING + 'maxSettings.conf') configFiles.append(statsPaths.STATSMONITORING + 'previousCrontab') configFiles.append(statsPaths.STATSMONITORING + 'previousFileChecksums') return configFiles
def getMachineParametersFromPreviousCall(): """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile(paths.STATSPREVIOUSMACHINEPARAMS): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS) return previousMachineParams
def previousUpdateFrequency(self): """ @summary : Finds and returns the frequency of the previous update. @return : The freqency of the previous update. """ paths = StatsPaths() paths.setPaths() lastUpdate = self.getTimeOfLastUpdateInLogs() fileName = paths.STATSTEMPAUTUPDTLOGS + str(lastUpdate).replace( " ", "_" ) lastUpdateFrequency = CpickleWrapper.load(fileName) return lastUpdateFrequency
def copySourceFiles( currentlyUsedLanguages ): """ @summary : makes sure source files are available in all of the source folder of each languages. """ statsPaths = StatsPaths() statsPaths.setPaths( 'en' ) englishSourceFolder = statsPaths.STATSWEBPAGES for language in currentlyUsedLanguages: if language != 'en' : statsPaths.setPaths( language ) destinationFolder = statsPaths.STATSWEBPAGES if not os.path.isdir( destinationFolder ) : os.makedirs( destinationFolder ) commands.getstatusoutput( "cp -r %s/* %s/" %( englishSourceFolder, destinationFolder ) )
def getListOfFileAccessFiles(): """ @summary: Returns the list of file version pickle files currently found on the local machine. """ statsPaths = StatsPaths() statsPaths.setPaths() listOfFileAccessFiles = [] if os.path.isdir(statsPaths.STATSLOGACCESS) : listOfFileAccessFiles = os.listdir( statsPaths.STATSLOGACCESS ) listOfFileAccessFiles = filter( filterentriesStartingWithDots, listOfFileAccessFiles ) listOfFileAccessFiles = [ statsPaths.STATSLOGACCESS + file for file in listOfFileAccessFiles ] return listOfFileAccessFiles
def __init__( self, accessDictionary = None, accessFile = "" ): """ @summary: LogFileAccessManager constructor. @param accessArrays: @param accessFile: """ paths = StatsPaths() paths.setPaths() if accessFile =="": accessFile = paths.STATSLOGACCESS + "default" self.accessDictionary = accessDictionary or {} # Empty array to start with. self.accessFile = accessFile #File that contains the current file acces. if self.accessDictionary == {} and os.path.isfile( self.accessFile ): self.loadAccessFile()
def copySourceFiles(currentlyUsedLanguages): """ @summary : makes sure source files are available in all of the source folder of each languages. """ statsPaths = StatsPaths() statsPaths.setPaths('en') englishSourceFolder = statsPaths.STATSWEBPAGES for language in currentlyUsedLanguages: if language != 'en': statsPaths.setPaths(language) destinationFolder = statsPaths.STATSWEBPAGES if not os.path.isdir(destinationFolder): os.makedirs(destinationFolder) commands.getstatusoutput("cp -r %s/* %s/" % (englishSourceFolder, destinationFolder))
def getParametersFromMonitoringConfigurationFile( self ): """ @summary : Gather all the parameters from the StatsPaths.STATSETC/config file. @return : All collected values in this order emails, machines, files, folders, maxUsages, errorsLogFile, maxSettingsFile. """ statsPaths = StatsPaths() statsPaths.setPaths() CONFIG = statsPaths.STATSETC +"monitoringConf" config = ConfigParser() if os.path.isfile( CONFIG ): file = open( CONFIG ) config.readfp( file ) self.emails = config.get( 'statsMonitoring', 'emails' ).split( ";" ) self.sender = config.get( 'statsMonitoring', 'sender' ) self.smtpServer = config.get( 'statsMonitoring', 'smtpServer' ) self.machines = config.get( 'statsMonitoring', 'machines' ).split( ";" ) self.files = config.get( 'statsMonitoring', 'files' ).split( ";" ) self.folders = config.get( 'statsMonitoring', 'folders' ).split( ";" ) self.maxUsages = config.get( 'statsMonitoring', 'maxUsages' ).split( ";" ) self.errorsLogFile = config.get( 'statsMonitoring', 'errorsLogFile' ) self.maxSettingsFile=config.get( 'statsMonitoring', 'maxSettingsFile' ) self.endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( time.time() ) ) self.startTime = self.getPreviousMonitoringJob(self.endTime) self.maximumGaps = self.getMaximumGaps( ) self.updateMachineNamesBasedOnExistingMachineTags() try: file.close() except: pass else: #print "%s configuration file not present. Please restore file prior to running" %CONFIG raise Exception( "%s configuration file not present. Please restore file prior to running" %CONFIG )
def __updateMaxSettingsFile( self, parameters ): """ @summary : This method is used to download the latest version of all the required files. @parameter parameters : StatsConfig parameters isntance """ paths = StatsPaths() paths.setPaths() if not os.path.isdir(paths.STATSMONITORING) : os.makedirs(paths.STATSMONITORING) if len( parameters.detailedParameters.uploadMachines ) != 0: machine = parameters.detailedParameters.uploadMachines[0] login = parameters.detailedParameters.uploadMachinesLogins[machine] commands.getstatusoutput( "scp %s@%s:%smaxSettings.conf %smaxSettings.conf >>/dev/null 2>&1" %(login, machine, paths.PDSCOLETC, paths.STATSMONITORING ) )
def __init__(self, accessDictionary=None, accessFile=""): """ @summary: LogFileAccessManager constructor. @param accessArrays: @param accessFile: """ paths = StatsPaths() paths.setPaths() if accessFile == "": accessFile = paths.STATSLOGACCESS + "default" self.accessDictionary = accessDictionary or { } # Empty array to start with. self.accessFile = accessFile #File that contains the current file acces. if self.accessDictionary == {} and os.path.isfile(self.accessFile): self.loadAccessFile()
def uploadGraphicFiles(parameters, machineParameters): """ @summary : Takes all the created daily graphics dedicated to clumbo and uploads them to the machines specified in the parameters. """ paths = StatsPaths() paths.setPaths() for uploadMachine in parameters.graphicsUpLoadMachines: output = commands.getoutput( "scp %s* %s@%s:%s " % (paths.STATSCOLGRAPHS, machineParameters.getUserNameForMachine(uploadMachine), uploadMachine, paths.PDSCOLGRAPHS)) print "scp %s* %s@%s:%s " % ( paths.STATSCOLGRAPHS, machineParameters.getUserNameForMachine(uploadMachine), uploadMachine, paths.PDSCOLGRAPHS) print output
def updateWebPages(generalParameters): """ @summary : Generates all the required web pages based on the language parameters found within the configuration files. """ paths = StatsPaths() paths.setPaths() generalParameters = StatsConfigParameters() generalParameters.getAllParameters() otherLanguages = [] generatorsTypes = [ DailyGraphicsWebPageGenerator, WeeklyGraphicsWebPageGenerator, MonthlyGraphicsWebPageGenerator, YearlyGraphicsWebPageGenerator, TotalsGraphicsWebPageGenerator ] for languagePair in generalParameters.webPagesLanguages: for generatorsType in generatorsTypes: generator = generatorsType(languagePair[0], languagePair[1]) generator.generateWebPage() topWebPageGenerator = TopWebPageGenerator(languagePair[0]) topWebPageGenerator.generateTopWebPage() otherLanguages.append(languagePair[0]) try: while (1): otherLanguages.remove(generalParameters.mainApplicationLanguage) except: pass bottomWebPageGenerator = BottomWebPageGenerator( generalParameters.mainApplicationLanguage, otherLanguages) bottomWebPageGenerator.printWebPage()