def getTimeParametersFromConfigurationFile(self): """ @summary: gathers all the time related parameters from the config file. @raise exception: Will raise and exception if one of the units of time used is illegal. """ readTimeUnits = [] paths = StatsPaths() paths.setBasicPaths() CONFIG = paths.STATSETC + "config" config = ConfigParser() file = open(CONFIG) config.readfp(file) self.pxStatsFrequency = {} self.monitoringFrequency = {} self.dbBackupsFrequency = {} self.pickleCleanerFrequency = {} self.generalCleanerFrequency = {} values = config.get('timeConfig', 'pxStatsFrequency').split('/') frequency, timeUnit = values[0], values[1] self.pxStatsFrequency[frequency] = timeUnit readTimeUnits.append(timeUnit) values = config.get('timeConfig', 'monitoringFrequency').split('/') frequency, timeUnit = values[0], values[1] self.monitoringFrequency[frequency] = timeUnit readTimeUnits.append(timeUnit) values = config.get('timeConfig', 'dbBackupsFrequency').split('/') frequency, timeUnit = values[0], values[1] self.dbBackupsFrequency[frequency] = timeUnit readTimeUnits.append(timeUnit) values = config.get('timeConfig', 'pickleCleanerFrequency').split('/') frequency, timeUnit = values[0], values[1] self.pickleCleanerFrequency[frequency] = timeUnit readTimeUnits.append(timeUnit) values = config.get('timeConfig', 'generalCleanerFrequency').split('/') frequency, timeUnit = values[0], values[1] self.generalCleanerFrequency[frequency] = timeUnit readTimeUnits.append(timeUnit) for unit in readTimeUnits: if unit not in TimeConfigParameters.validTimeUnits: raise Exception( "Invalid time unit found in configuration file.") self.dailyWebPageFrequency = config.get( 'timeConfig', 'dailyWebPageUpdatesFrequency').replace( " ", "").replace("'", "").replace('"', '') self.weeklyWebPageFrequency = config.get( 'timeConfig', 'weeklyWebPageUpdatesFrequency').replace( " ", "").replace("'", "").replace('"', '') self.monthlyWebPageFrequency = config.get( 'timeConfig', 'monthlyWebPageUpdatesFrequency').replace(" ", "").replace( "'", "").replace('"', '') self.yearlyWebPageFrequency = config.get( 'timeConfig', 'yearlyWebPageUpdatesFrequency').replace( " ", "").replace("'", "").replace('"', '') self.totalWebPagesUpdatesFrequency = config.get( 'timeConfig', 'totalWebPagesUpdatesFrequency').replace( " ", "").replace("'", "").replace('"', '') try: file.close() except: pass
""" - Small function that adds pxStats to sys path. """ sys.path.insert(1, os.path.dirname(os.path.abspath(__file__)) + '/../../') from pxStats.lib.StatsPaths import StatsPaths from pxStats.lib.StatsPickler import StatsPickler from pxStats.lib.StatsDateLib import StatsDateLib from pxStats.lib.PickleMerging import PickleMerging from pxStats.lib.GnuPlotter import GnuPlotter from pxStats.lib.GeneralStatsLibraryMethods import GeneralStatsLibraryMethods from pxStats.lib.Translatable import Translatable """ These imports require pxlib """ statsPaths = StatsPaths() statsPaths.setBasicPaths() sys.path.append(statsPaths.PXLIB) import logging from Logger import Logger LOCAL_MACHINE = os.uname()[1] CURRENT_MODULE_ABS_PATH = os.path.abspath(__file__).replace(".pyc", ".py") class GnuGraphicProducer(Translatable): def __init__( self, directory, fileType, clientNames = None , groupName = "", timespan = 12,\ currentTime = None, productTypes = None, logger = None, logging = True, machines = None,\
def printWebPage( self ): """ @summary : prints out the entire bottom web page @precondition: Requires _ translator to have been set prior to calling this function. """ global _ paths = StatsPaths() paths.setPaths( LanguageTools.getMainApplicationLanguage() ) fileName = paths.STATSWEBPAGES + "bottom.html" fileHandle = open( fileName, "w" ) fileHandle.write(""" <html> <head> """) self.__printJavaScript(fileHandle) fileHandle.write(""" </head> """) fileHandle.write(""" <body bgcolor="#FFD684"> <div style="position:absolute;top:20%%;vertical-align: middle;text-align:center;left:15%%;bottom:0%%;"> <img name="logo" id="logo" src="images/mainLogo_%s.gif" ></img> </div> """ %self.mainLanguage) fileHandle.write( """ <div name="linksSection" id="linksSection" style="position:absolute;top:67%;vertical-align: middle;text-align:center;left:45%;bottom:0%;"> """) for i in range( len( self.otherLanguages ) ): _ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.otherLanguages[i] ) try: fileHandle.write("""<a href="top_%s.html" target="top" onclick="JavaScript:%sVersionWasClicked()">"""%( self.otherLanguages[i],self.otherLanguages[i]) + _("English version.")+ """</a>""") except: print _( "Error.Unsupported language detected." ) print _( "Make sure %s is a supported language") %( self.otherLanguages[i] ) print _( "Program terminated") sys.exit() if i != len(self.otherLanguages)-1 : fileHandle.write( "<br>" ) fileHandle.write( """ </div> </body> </html> """) fileHandle.close()
import os, sys, string """ Small method that adds pxStats to syspath. """ sys.path.insert(1, os.path.dirname(os.path.abspath(__file__)) + '/../../') from pxStats.lib.StatsPaths import StatsPaths from pxStats.lib.StatsConfigParameters import StatsConfigParameters from pxStats.lib.MachineConfigParameters import MachineConfigParameters from pxStats.lib.LanguageTools import LanguageTools from pxStats.lib.Translatable import Translatable """ - Small function that adds pxLib to sys path. """ STATSPATHS = StatsPaths() STATSPATHS.setPaths() sys.path.append(STATSPATHS.PXLIB) from PXManager import * LOCAL_MACHINE = os.uname()[1] CURRENT_MODULE_ABS_PATH = os.path.abspath(__file__).replace(".pyc", ".py") class TopWebPageGenerator(Translatable): def __init__(self, outputLanguage): """ @param outputLanguage: Language that will be displayed on the web page.
def __createTheWebPage(self, machineTags, supportedLanguages): """ @summary : Generates the top.html web page to be displayed as the top frame of the pxstats web site. @param machineTags : Tags representing machine groups for which we are producing graphics. @param supportedLanguages : list of languages supported by the application @precondition: Requires _ translator to have been set prior to calling this function. """ global _ paths = StatsPaths() paths.setPaths(self.outputLanguage) file = "%stop_%s.html" % (paths.STATSWEBPAGES, self.outputLanguage) fileHandle = open(file, 'w') languageOptions = """<option value="">%s</option>\n""" % self.outputLanguage for language in supportedLanguages: if language != self.outputLanguage: languageOptions = languageOptions + """<option value="">%s</option>\n""" % language fileHandle.write(""" <html> <style type="text/css"> div.left { float: left; } div.right {float: right; } </style> <script type="text/javascript"> function gup( name ){ name = name.replace(/[\[]/,"\\\[").replace(/[\]]/,"\\\]"); var regexS = "[\\?&]"+name+"=([^&#]*)"; var regex = new RegExp( regexS ); var results = regex.exec( window.location.href ); if( results == null ) return ""; else return results[1]; } var lastLinkedClicked = ""; lastLinkedClicked= gup('lastLinkedClicked'); function applyvalue(value){ lastLinkedClicked = value; } function callNewTopWebPage(){ parent.top.location.href= 'top_' + document.getElementById('language')[document.getElementById('language').selectedIndex].text + '.html' +'?lastLinkedClicked='+lastLinkedClicked; } </script> <body text="white" link="white" vlink="white" bgcolor="#3366CC" > <div class="left"> """ + _("Language") + """ <select class="dropDownBox" name="language" id="language" OnChange="javascript:callNewTopWebPage();" > """ + languageOptions + """ </select> """ + _("Individual graphics") + """ : <a href="html_%s/dailyGraphs_%s.html" target="bottom" onclick="javascript:applyvalue('href1')" id="href1" name="href1" class="links" >""" % (self.outputLanguage, self.outputLanguage) + _("Daily") + """</a> <a href="html_%s/weeklyGraphs_%s.html" target="bottom" onclick="javascript:applyvalue('href2')" id='href2'>""" % (self.outputLanguage, self.outputLanguage) + _("Weekly") + """</a> <a href="html_%s/monthlyGraphs_%s.html" target="bottom" onclick="javascript:applyvalue('href3')" id='href3'>""" % (self.outputLanguage, self.outputLanguage) + _("Monthly") + """</a> <a href="html_%s/yearlyGraphs_%s.html" target="bottom" onclick="javascript:applyvalue('href4')" id='href4'>""" % (self.outputLanguage, self.outputLanguage) + _("Yearly") + """</a> """) i = 5 if machineTags != []: fileHandle.write(""" """ + _("Clusters") + """ : """) for machineTag in machineTags: fileHandle.write(""" <a href="html_%s/%s_%s.html" target="bottom" id='href%s' onclick="javascript:applyvalue('href%s')">%s</a> """ % (self.outputLanguage, machineTag.replace(',', ''), self.outputLanguage, i, i, string.upper(machineTag))) i = i + 1 fileHandle.write(""" </div> <div class="right"> <a href="html_%s/archives" target="bottom" id='href%s' onclick="javascript:applyvalue('href%s')">Archives</a> """ % (self.outputLanguage, (i + 1), (i + 1)) + """ <a href="../scripts/cgi-bin/graphicsRequestPage.py?lang=%s" target="bottom" id='href%s' onclick="javascript:applyvalue('href%s')">""" % (self.outputLanguage, (i + 2), (i + 2)) + _("Requests") + """</a> <a href="html_%s/helpPages/glossary_%s.html" target="bottom" id='href%s' onclick="javascript:applyvalue('href%s')">""" % (self.outputLanguage, self.outputLanguage, (i + 3), (i + 3)) + _("Glossary") + """</a> <a href="html_%s/docPages/listOfDocumentationFiles_%s.html" target="bottom" id='href%s' onclick="javascript:applyvalue('href%s')">""" % (self.outputLanguage, self.outputLanguage, (i + 4), (i + 4)) + _("Documentation") + """ </a> </div> <script type="text/javascript"> if( lastLinkedClicked != "" ){ link = document.getElementById(lastLinkedClicked).href; parent.bottom.location.href = link; } </script> </body> </html> """) fileHandle.close()
def createCopy( self, copyToArchive = True , copyToColumbo = True ): """ @summary : Creates a copy of the created image file so that it easily be used in px's columbo or other daily image program. @summary : If copies are to be needed for graphcis other than daily graphs, please modify this method accordingly! """ statsPaths = StatsPaths() statsPaths.setPaths(self.outputLanguage) _ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.outputLanguage ) src = self.imageName if self.groupName != "": clientName = self.groupName else: clientName = "" if len( self.clientNames ) == 0: clientName = self.clientNames[0] else: for name in self.clientNames : clientName = clientName + name if name != self.clientNames[ len(self.clientNames) -1 ] : clientName = clientName + "-" StatsDateLib.setLanguage( self.outputLanguage ) # Makes sure month is in the right language. year, month, day = StatsDateLib.getYearMonthDayInStrfTime( StatsDateLib.getSecondsSinceEpoch( self.currentTime ) - 60 ) # -60 means that a graphic ending at midnight StatsDateLib.setLanguage( self.workingLanguage )# Sets language back to working language. # would be named after the rpevious day. if copyToArchive == True : destination = statsPaths.STATSGRAPHSARCHIVES + _("daily/%s/%s/") %( self.fileType, clientName ) + str(year) + "/" + str(month) + "/" + str(day) + ".png" if not os.path.isdir( os.path.dirname( destination ) ): os.makedirs( os.path.dirname( destination ), 0777 ) dirname = os.path.dirname( destination ) while( dirname != statsPaths.STATSGRAPHSARCHIVES[:-1] ):#[:-1] removes the last / character try: os.chmod( dirname, 0777 ) except: pass dirname = os.path.dirname(dirname) shutil.copy( src, destination ) try: os.chmod( destination, 0777 ) except: pass #print "cp %s %s " %( src, destination ) if copyToColumbo == True : destination = statsPaths.STATSGRAPHS + _("webGraphics/columbo/%s_%s.png") %(clientName,self.outputLanguage) if not os.path.isdir( os.path.dirname( destination ) ): os.makedirs( os.path.dirname( destination ), 0777 ) os.chmod( os.path.dirname( destination ), 0777 ) shutil.copy( src, destination ) try: os.chmod( destination, 0777 ) except: pass
def printWebPages(self): """ @summary : prints out the entire list of doc pages required by all languages. @precondition: Requires _ translator to have been set prior to calling this function. """ global _ statsPaths = StatsPaths() statsPaths.setPaths(self.mainLanguage) for language in self.languages: _ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH) fileName = statsPaths.STATSWEBPAGESHTML + "docPages/" + "listOfDocumentationFiles_%s.html" % ( language) if not os.path.isdir(os.path.dirname(fileName)): os.makedirs(os.path.dirname(fileName)) fileHandle = open(fileName, "w") fileHandle.write(""" <html> <head> """) fileHandle.write(""" </head> """) fileHandle.write(""" <body bgcolor="#FFD684"> <h3><u>%s</u></h3> """ % (_("List of available documentation files:"))) availableDocFiles = self.__getDocFilesToLinkTo(language) fileHandle.write(""" <div style="position:absolute;top:20%%;vertical-align: middle;text-align:center;left:15%%;bottom:0%%;"> <img name="logo" id="logo" src="images/docFilesLogo_%s.gif" ></img> </div> """ % (language)) for file in availableDocFiles: fileHandle.write(""" <a href="docPages/%s" target="bottom">""" % (file) + str(file).replace("_%s.html" % (language), "") + """</a> <br> """) fileHandle.write(""" </body> """) fileHandle.write(""" </html> """)
def generateWebPage( images, lang ): """ @summary : Generates a web page that simply displays a series of images one on top of the other. @param images : List of images to display. @param lang : language with whom this generator was called. """ smallImageWidth = 900 smallImageHeight = 320 statsPaths = StatsPaths() statsPaths.setPaths( lang ) file = statsPaths.STATSWEBPAGESHTML + "combinedImageWebPage.html" fileHandle = open( file, "w") fileHandle.write( """ <html> <head> <style type="text/css"> a.photosLink{ display: block; width: 1200px; height: 310px; background: url("") 0 0 no-repeat; text-decoration: none; } </style> <script type="text/javascript" src="../scripts/js/windowfiles/dhtmlwindow.js"> This is left here to give credit to the original creators of the dhtml script used for the group pop ups: /*********************************************** * DHTML Window Widget- Dynamic Drive (www.dynamicdrive.com) * This notice must stay intact for legal use. * Visit http://www.dynamicdrive.com/ for full source code ***********************************************/ </script> <script> counter =0; function wopen(url, name, w, h){ // This function was taken on www.boutell.com w += 32; h += 96; counter +=1; var win = window.open(url, counter, 'width=' + w + ', height=' + h + ', ' + 'location=no, menubar=no, ' + 'status=no, toolbar=no, scrollbars=no, resizable=no'); win.resizeTo(w, h); win.focus(); } function transport( image ){ wopen( image, 'popup', %s, %s); } </script> </head> <body> """ %( smallImageWidth, smallImageHeight ) ) relativePathTowardsPxStats = "../../../pxStats/" for i in range(len(images) ): pathTowardsImage = str(images[i]).split( "pxStats" )[-1:][0] fileHandle.write(""" <a href="#" class="photosLink" name="photo%s" onclick="javascript:transport('%s')" id="photo%s" border=0> </a> <script> document.getElementById('photo%s').style.background="url(" + "%s" + ") no-repeat"; </script> """%( i, relativePathTowardsPxStats + pathTowardsImage, i, i, relativePathTowardsPxStats + pathTowardsImage ) ) fileHandle.write( """ </body> </html> """) fileHandle.close() try: os.chmod(file,0777) except: pass
def buildImageName( self ): """ @summary : Builds and returns the absolute fileName so that it can be saved If folder to file does not exists creates it. """ statsPaths = StatsPaths() statsPaths.setPaths(self.outputLanguage) _ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.workingLanguage ) entityName = "" if len( self.clientNames ) == 0: entityName = self.clientNames[0] else: if self.groupName == "" : for name in self.clientNames : entityName = entityName + name if name != self.clientNames[ len(self.clientNames) -1 ] : entityName = entityName + "-" else: entityName = self.groupName date = self.currentTime.replace( "-","" ).replace( " ", "_") if self.productTypes[0] == _("All") or self.productTypes[0] == "*": formattedProductName = LanguageTools.translateTerm(_("All"), self.workingLanguage, self.outputLanguage, CURRENT_MODULE_ABS_PATH) else: combinedProductsName = "" for product in self.productTypes: combinedProductsName = combinedProductsName + str(product) + "_" formattedProductName = combinedProductsName[ :-1 ] #remove trailing _ character. _ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.outputLanguage ) folder = statsPaths.STATSGRAPHS + _("others/gnuplot/%.50s/") %( entityName ) translatedStatsTypes = [ LanguageTools.translateTerm(statsType, self.workingLanguage, self.outputLanguage, CURRENT_MODULE_ABS_PATH)\ for statsType in self.statsTypes ] fileName = folder + _("%s_%s_%s_%s_%shours_on_%s_for_%s_products.png") %( self.fileType, entityName, date, translatedStatsTypes,\ self.timespan, self.machines, formattedProductName ) fileName = fileName.replace( '[', '').replace(']', '').replace(" ", "").replace( "'","" ) if not os.path.isdir( folder ): os.makedirs( folder, 0777 ) os.chmod(folder, 0777) if len( os.path.basename(fileName) ) > (os.statvfs( folder )[statvfs.F_NAMEMAX]): # length of file too long maximumLength = (os.statvfs( folder )[statvfs.F_NAMEMAX]) - ( 30 + len(date) + len( str(translatedStatsTypes)) + len( str( self.timespan ) ) ) maxIndyLength = maximumLength / 3 #reduce entityname, machine names and products wich are the only parts wich can cause us to bust the maximum filename size. fileName = folder + ( "%s_%." + str( maxIndyLength )+ _("s_%s_%s_%shours_on_%.") + str( maxIndyLength ) + \ _("s_for_%.") + str( maxIndyLength ) + _("s_products.png") ) \ %( self.fileType, entityName, date, translatedStatsTypes, self.timespan,\ self.machines, formattedProductName ) return fileName
def generateWebPage(sourlientNames, groups, fileType, outputFileName, language): """ @summary: Generates popupAdder web page named after the received outputFileName and based on the list of sourlients names received @param sourlientNames : List of sources or clients that need to be printed. @param groups : List of groups that need to be printed. @param fileType: tx or rx @param outputFileName : Filename that needs to be created. @return : None """ statsPaths = StatsPaths() statsPaths.setPaths(language) _ = LanguageTools.getTranslatorForModule(CURRENT_MODULE_ABS_PATH, language) if not os.path.isdir(os.path.dirname(outputFileName)): os.makedirs(os.path.dirname(outputFileName)) """ Redirect the output""" #print outputFileName fileHandle = open(outputFileName, "w") oldStdOut = sys.stdout #save previous stdout sys.stdout = fileHandle print """ <html> <head> <title>Add items to list.</title> <meta name="Author" content="Nicholas Lemay"> <meta name="Description" content="Small popup window used to add items into a list. To be used with graphicsResquests.py"> <meta name="Keywords" content=""> <style type="text/css"> div.selectObject{ width:300px; height: auto; } </style> <link rel="stylesheet" type="text/css" href="/css/style.css"> <script type="text/javascript" language="JavaScript"> function popupAddingWindow( url ) { var newWindow; var props = 'scrollBars=no,resizable=no,toolbar=no,menubar=no,location=no,directories=no,width=700,height=300'; newWindow = window.open(url, "Add_from_Src_to_Dest", props); } function closeWindow(){ window.close(); } // Fill the selcted item list with the items already present in parent. function copyLists( srcList, destList ) { var len = destList.length; for(var i = 0; i < srcList.length; i++) { if ( srcList.options[i] != null ) { //Check if this value already exist in the destList or not //if not then add it otherwise do not add it. var found = false; for(var count = 0; count < len; count++) { if (destList.options[count] != null) { if (srcList.options[i].text == destList.options[count].text) { found = true; break; } } } if (found != true) { destList.options[len] = new Option(srcList.options[i].text); len++; } } } } // Add the SELECTED items from the source to destination list // will only add the items wich are not allready present in dest list. function addSrcToDestList( srcList, destList ) { var len = destList.length; for(var i = 0; i < srcList.length; i++) { if ((srcList.options[i] != null) && (srcList.options[i].selected)) { //Check if this value already exist in the destList or not //if not then add it otherwise do not add it. var found = false; for(var count = 0; count < len; count++) { if (destList.options[count] != null) { if (srcList.options[i].text == destList.options[count].text) { found = true; break; } } } if (found != true) { destList.options[len] = new Option(srcList.options[i].text); len++; } } } } // Deletes from the destination list. function deleteFromList( list ) { var len = list.options.length; for(var i = (len-1); i >= 0; i--) { if ((list.options[i] != null) && (list.options[i].selected == true)) { list.options[i] = null; } } } </script> </head> <body text="#FFFFFF" link="#FFFFFF" vlink="000000" bgcolor="#7ACC7A"> <center> <form name="adderForm" method="POST"> <table bgcolor="#FFF4E5" > <tr> <font color ="white"> <td bgcolor="#006699" width="300" >""" + _( "Available") + """</td> <td bgcolor="#006699" > </td> <td bgcolor="#006699" width="300" >""" + _( "Selected") + """</td> </font> </tr> <tr> <td bgcolor="#7ACC7A" width="300"> <select size="12" style="width: 300px;height: 225px;font: 14px;" name="srcList" multiple> """ startingIndex = 1 if len(groups) > 0: print """ <optgroup label=""" + '"' + _( "Groups:") + """">""" + _("Groups:") + """</optgroup> """ for i in range(len(groups)): print """ <option value="%s">%s</option> """ % (i + startingIndex, groups[i]) startingIndex = i if len(sourlientNames) > 0: if fileType == "tx": print """ <optgroup label=""" + '"' + _( "TX clients :") + """">""" + _( "TX clients :") + """ </optgroup> """ elif fileType == "rx": print """ <optgroup label=""" + '"' + _( "RX sources :") + """">""" + _( "RX sources :") + """ </optgroup> """ else: print """ <optgroup label=""" + '"' + _( "Sourlients :") + """">""" + _( "Sourlients :") + """ </optgroup> """ for i in range(len(sourlientNames)): print """ <option value="%s">%s</option> """ % (i + startingIndex, sourlientNames[i]) print """ </select> </td> <td bgcolor="#FFF4E5" width="74" align="center"> <input type="button" value=" >> " style="font: 14px;" onClick="javascript:addSrcToDestList( document.forms['adderForm'].elements['srcList'], document.forms['adderForm'].elements['destList'] )"> <br><br> <input type="button" value=" << " style="font: 14px;" onclick="javascript:deleteFromList( document.forms['adderForm'].elements['destList'] );"> <br><br> <input type="button" value=""" + '"' + _( "Done" ) + '"' + """ style="font: 14px;" onClick ="javascript:window.opener.copyLists(document.forms['adderForm'].elements['destList'], window.opener.document.forms['inputForm'].elements['sourlientList']);javascript:closeWindow();"> </td> <td bgcolor="#7ACC7A" width="300"> <select size="12" style="width: 300px;height: 225px;font: 14px;" name="destList" multiple> </select> </td> </tr> </table> </form> </body> </html> """ fileHandle.close() sys.stdout = oldStdOut #resets standard output
import os, sys import cgi import cgitb; cgitb.enable() """ Small function that adds pxStats to the sys path. """ sys.path.insert(1, sys.path[0] + '/../../..') from pxStats.lib.StatsPaths import StatsPaths from pxStats.lib.LanguageTools import LanguageTools """ Small method required to add pxLib to syspath. """ PATHS = StatsPaths() PATHS.setBasicPaths() sys.path.append( PATHS.PXLIB ) def returnReplyToQuerier( error ="" ): """ @summary : Prints an empty reply so that the receiving web page will not modify it's display. @param error : Error to return to querier. @return : None @note: Method does not actually "return" anything.
def main(): """ @summary : Generates the web page based on the received machines and file type parameters. """ try: newForm = {} form = cgi.FieldStorage() for key in form.keys(): value = form.getvalue(key, "") if isinstance(value, list): # Multiple username fields specified newvalue = ",".join(value) else: newvalue = value newForm[key.replace("?", "")] = newvalue form = newForm try: language = form["lang"] except: language = "" error = "Error.Unknwown language detected in popSourlientUpAdder" try: fileType = form['fileType'] if fileType != 'tx' and fileType != 'rx': error = "Error. File type needs to be either rx or tx." except: fileType = "" try: machines = form['machines'] machine = machines.split(',')[0] machines = machines.replace(',', '') except: error = "Error. Machine names need to be specified." machine = "" if machines != "": rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machine) groups = getGroups(fileType, machine) #print "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language ) #----------------------------- fileHandle = open("/web/pxStats/out","w") # fileHandle.write(sys.path[0] + "/../../html/popUps/%s%sPopUpSourlientAdder_%s.html" %( fileType, machines, language )) #---------------------------------------------------- fileHandle.close() path = StatsPaths() path.setPaths(language) if fileType == "tx": generateWebPage( txNames, groups, fileType, path.STATSWEBPAGESHTML + "/popUps/%s%sPopUpSourlientAdder_%s.html" % (fileType, machines, language), language) elif fileType == "rx": generateWebPage( rxNames, groups, fileType, path.STATSWEBPAGESHTML + "/popUps/%s%sPopUpSourlientAdder_%s.html" % (fileType, machines, language), language) returnReply('') except Exception, instance: fileHandle = open("/web/pxStats/out", "w") fileHandle.write("bug in main method") fileHandle.write(str(instance)) fileHandle.close() sys.exit()
def createSymbolicLinks(path, currentlyUsedLanguages): """ @summary : create symbolic links from web-interface to general pxStats package. This will prevent us from having to sync both sides all the time. i.e updating pxStats ( via svn update for example) will update both the web interface and the source files at the same time. @param path : Paths in which we are installing the web interface. @param currentlyUsedLanguages: Languages currently set to be displayed in the web interface @precondition : copyFiles method MUST have been called prior to calling this method. """ statsPaths = StatsPaths() #Links to files in the main application language. statsPaths.setPaths(LanguageTools.getMainApplicationLanguage()) #index.html commands.getstatusoutput("ln -s %s/index.html %s/index.html" % (statsPaths.STATSWEBPAGES, path)) #print "ln -s %s/index.html %s/index.html" %( statsPaths.STATSWEBPAGES, path ) # .../path/bottom.html Only on, multilingual fomr of this file exists. commands.getstatusoutput("ln -s %s/bottom.html %s/bottom.html" % (statsPaths.STATSWEBPAGES, path)) #print "ln -s %s/bottom.html %s/bottom.html" %(statsPaths.STATSWEBPAGES , path ) # .../path/bottom.html Only on, multilingual fomr of this file exists. commands.getstatusoutput( "ln -s %s/top_%s.html %s/top.html" % (statsPaths.STATSWEBPAGES, LanguageTools.getMainApplicationLanguage(), path)) #print "ln -s %s/bottom.html %s/bottom.html" %(statsPaths.STATSWEBPAGES , path ) # .../path/pxStats commands.getstatusoutput("ln -s %s %s/pxStats" % (statsPaths.STATSROOT, path)) #print "ln -s %s %s/pxStats" %( statsPaths.STATSROOT, path ) #.../path/images commands.getstatusoutput("ln -s %s/images %s/images" % (statsPaths.STATSWEBPAGES, path)) #print "ln -s %s/images %s/images" %( statsPaths.STATSWEBPAGES, path ) #.../path/scripts/cgi-bin commands.getstatusoutput("ln -s %s %s/scripts/cgi-bin " % (statsPaths.STATSWEBPAGESGENERATORS, path)) #print "ln -s %s %s/scripts/cgi-bin "%( statsPaths.STATSWEBPAGESGENERATORS, path ) for language in currentlyUsedLanguages: statsPaths.setPaths(language) # .../path/html_lang commands.getstatusoutput("ln -s %s/html %s/html_%s" % (statsPaths.STATSWEBPAGES, path, language)) #print "ln -s %s/html %s/html_%s" %( statsPaths.STATSWEBPAGES, path, language ) # .../path/archives_lang commands.getstatusoutput( "ln -s %s %s/archives_%s" % (statsPaths.STATSGRAPHSARCHIVES[:-1], path, language)) #print "ln -s %s %s/archives_%s" %( statsPaths.STATSGRAPHSARCHIVES[:-1], path, language ) # .../paths/html_lang/archives commands.getstatusoutput( "ln -s %s %s/html_%s/archives" % (statsPaths.STATSGRAPHSARCHIVES[:-1], path, language)) #print "ln -s %s %s/html_%s/archives" %( statsPaths.STATSGRAPHSARCHIVES[:-1], path, language ) #.../paths/html_lang/csvFiles commands.getstatusoutput( "ln -s %s %s/html_%s/%s" % (statsPaths.STATSCSVFILES[:-1], path, language, os.path.basename(statsPaths.STATSCSVFILES))) # .../path/top_lang.html commands.getstatusoutput( "ln -s %s/top_%s.html %s/top_%s.html" % (statsPaths.STATSWEBPAGES, language, path, language)) #print "ln -s %s/top_%s.html %s/top_%s.html" %( statsPaths.STATSWEBPAGES, language, path, language ) #.../path/scripts/js_lang commands.getstatusoutput("ln -s %s/js %s/scripts/js_%s " % (statsPaths.STATSWEBPAGES, path, language)) #print "ln -s %s/js %s/scripts/js_%s " %( statsPaths.STATSWEBPAGES, path, language ) commands.getstatusoutput( "ln -s %s/html/howTo_%s.html %s/html_%s/docPages/links/howTo_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/installation_%s.html %s/html_%s/docPages/links/installation_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/monitoringDoc_%s.html %s/html_%s/docPages/links/monitoringDoc_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/translationDoc_%s.html %s/html_%s/docPages/links/translationDoc_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/developersDoc_%s.html %s/html_%s/docPages/links/developersDoc_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/installation_%s.html %s/html_%s/docPages/links/installation_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/rrdToolDoc_%s.html %s/html_%s/docPages/links/rrdToolDoc_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/troubleshooting_%s.html %s/html_%s/docPages/links/troubleshooting_%s.html" % (statsPaths.STATSDOC, language, path, language, language)) commands.getstatusoutput( "ln -s %s/html/images %s/html_%s/docPages/links/images" % (statsPaths.STATSDOC, path, language))
def updateDatabases(parameters, machineParameters, currentTimeInIsoFormat): """ @summary : Updates all the required databases by transferring the data found in the pickle files into rrd databases files. First transfers all the pickles into databases for all the clusters. Then combines all the data required by the different groups found within the config file. @param parameters: StatsConfigParameters instance containing the parameters found in the config file. @param machineParameters: MachineConfigParameters instance containing the parameters found in the config file. @param currentTimeInIsoFormat : Time at which this program was originally called. @return : None """ paths = StatsPaths() paths.setPaths() #Small safety measure in case another instance of the program is allready running. if transferToDatabaseAlreadyRunning() == False: for tag in parameters.machinesToBackupInDb: machines = machineParameters.getMachinesAssociatedWith(tag) machines = str(machines).replace("[", "").replace("]", "").replace(" ", "") output = commands.getoutput( "%stransferPickleToRRD.py -m '%s' -e '%s' " % (paths.STATSBIN, machines, currentTimeInIsoFormat)) print "%stransferPickleToRRD.py -m '%s' " % (paths.STATSBIN, machines) print "output:%s" % output if parameters.groupParameters.groups != []: for group in parameters.groupParameters.groups: groupMembers = str( parameters.groupParameters.groupsMembers[group]).replace( "[", "").replace("]", "").replace(" ", "") groupMachines = str( parameters.groupParameters.groupsMachines[group]).replace( "[", "").replace("]", "").replace(" ", "") groupProducts = str( parameters.groupParameters.groupsProducts[group]).replace( "[", "").replace("]", "").replace(" ", "") groupFileTypes = str( parameters.groupParameters.groupFileTypes[group]).replace( "[", "").replace("]", "").replace(" ", "") output = commands.getoutput( "%stransferPickleToRRD.py -c '%s' -m '%s' -g '%s' -f %s -p '%s' -e '%s' " % (paths.STATSBIN, groupMembers, groupMachines, group, groupFileTypes, groupProducts, currentTimeInIsoFormat)) print "%stransferPickleToRRD.py -c '%s' -m '%s' -g '%s' -f %s -p '%s' " % ( paths.STATSBIN, groupMembers, groupMachines, group, groupFileTypes, groupProducts) print output
def updatePickles(parameters, machineParameters, currentTimeInIsoFormat): """ @summary : Updates the pickle files for all the specified log machines so that they are available for graphic production. @note : Pickling is to be done on specified pickling machines. All the pickle files that are produced on remote machines will be downloaded on the local machine. @param parameters: StatsConfigParameters instance containing the parameters found in the config file. @param machineParameters: MachineConfigParameters instance containing the parameters found in the config file. @param currentTimeInIsoFormat : Time at which this program was originally called. """ nbChildProcess = 0 paths = StatsPaths() paths.setPaths() for tag in parameters.sourceMachinesTags: pid = os.fork() if pid == 0: #if child sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for i in range(len(sourceMachines)): picklingMachine = parameters.detailedParameters.picklingMachines[ tag][i] # If pickling and source machines differ, download log files frm source to pickling machine. if sourceMachines[i] != picklingMachine: if parameters.detailedParameters.picklingMachines[tag][ i] != LOCAL_MACHINE: #pickling to be done elsewhere for j in range( 3 ): #do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) output = commands.getoutput( "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " % (machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i])) print "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i]) print output else: for j in range( 3 ): #do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) print "le remote paths : ", remotePxLogPath print output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i])) print print "*****************************" print print "rsync -avzr --delete-before -e ssh " + """%s@%s:%s %s%s/""" % ( machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i]) print print output if picklingMachine != LOCAL_MACHINE: #pickling to be done elsewhere,needs ssh output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f rx --date "%s" ' """ % (machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f rx' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f tx --date "%s" ' """ ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f tx' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """%spickleSynchroniser.py -l %s -m %s """ % (paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine), picklingMachine)) print "%spickleSynchroniser.py -l %s -m %s " % ( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine), picklingMachine) print output else: # pickling is to be done locally. Log files may or may not reside elsewhere. output = commands.getoutput( """python %spickleUpdater.py -f rx -m %s --date "%s" """ % (paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "python %spickleUpdater.py -f rx -m %s " % ( paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """python %spickleUpdater.py -f tx -m %s --date "%s" """ % (paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "python %spickleUpdater.py -f tx -m %s " % ( paths.STATSBIN, sourceMachines[i]) print output sys.exit() elif nbChildProcess != 0 and nbChildProcess % 3 == 0: while True: #wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait() except: break while True: #wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait() except: break
def main(): """ @summary: Small test case to see if everything works out well. @note: IMPORTANT if you modifiy this file, run this method to make sure it still passes all the tests. If test are no longer valid, please modify accordingly. """ from LogFileAccessManager import LogFileAccessManager paths = StatsPaths() paths.setPaths() # # Create text file for testing. # testDirectory = paths.STATSDATA + "logFileAccessTestFolder/" if not os.path.isdir(testDirectory): os.makedirs(testDirectory) testTextfile = testDirectory + "testTextfile" fileHandle = open(testTextfile, 'w') old_stdout = sys.stdout #redirect standard output to the file sys.stdout = fileHandle for i in range(100): print "%s-A line written for testing." % i fileHandle.close() sys.stdout = old_stdout #resets standard output # #Read file like normal file and stop in the middle. # fileHandle = open(testTextfile, 'r') for i in range(50): fileHandle.readline() lastReadPosition = fileHandle.tell() fileHandle.close() # # Set LogFileAccessManager with the previous infos. # testFile = testDirectory + "testLFAMfile" lfam = LogFileAccessManager(accessFile=testFile) firstLine = lfam.getFirstLineFromFile(testTextfile) lfam.setFirstLineAndLastReadPositionAssociatedwith(firstLine, lastReadPosition, "testId") # # Unit-like test every method to make sure the result is what is expected. # Section for getters. # if firstLine != "0-A line written for testing.\n": print "getFirstLineFromFile is corrupted. Please repair " if lfam.getFirstLineAndLastReadPositionAssociatedwith("testId") != ( "0-A line written for testing.\n", 1540): print "getFirstLineAndLastReadPositionAssociatedwith is corrupted. Please repair." if lfam.getLastReadPositionAssociatedWith("testId") != 1540: print "getLastReadPositionAssociatedWith is corrupted. Please repair." # # Section for testing Setters # lfam.setFirstLineAssociatedwith("firstLine", 'testId') if lfam.getLineAssociatedWith('testId') != 'firstLine': print "setFirstLineAssociatedwith is corrupted. Please repair." lfam.setLastReadPositionAssociatedwith(18987, 'testId') if lfam.getLastReadPositionAssociatedWith('testId') != 18987: print "setLastReadPositionAssociatedwith is corrupted. Please repair." lfam.setFirstLineAndLastReadPositionAssociatedwith("testline2", 1285647, 'testId') if lfam.getFirstLineAndLastReadPositionAssociatedwith('testId') != ( "testline2", 1285647): print "setFirstLineAndLastReadPositionAssociatedwith is corrupted. Please repair." lfam.saveAccessDictionary() lfam.loadAccessFile() if lfam.getFirstLineAndLastReadPositionAssociatedwith('testId') != ( "testline2", 1285647): print "saveAccessDictionary and/or loadAccessFile is corrupted. Please repair." print "Testing done."